Spaces:
Running
Running
| # %% | |
| # TODOS: Plots with plotly | |
| import json | |
| import pandas as pd | |
| import streamlit as st | |
| import plotly.express as px | |
| from config import other_info_dict | |
| from utils import * | |
| # %% | |
| st.title("LLM assessments: Microsoft's Phi-2 and Google's Gemma-7b") | |
| # st.image('model_card.png', caption='Hugging face description', use_column_width=True) | |
| import urllib.request | |
| import os | |
| llm_options = [ | |
| "Microsoft's Phi-2", | |
| "Google's Gemma-7b" | |
| ] | |
| model_select_options = st.multiselect( | |
| 'Select one or more models:', | |
| llm_options, | |
| [llm_options[0]]) | |
| prefix_post_processing_dict = {} | |
| st.markdown("---") | |
| for t_opt in model_select_options: | |
| st.write(t_opt) | |
| if t_opt == "Microsoft's Phi-2": | |
| prefix_post_processing_dict[t_opt] = os.environ["POST_PROCESSING_JSON"] | |
| st.write(""" | |
| Microsoft's Phi-2 (https://huggingface.co/microsoft/phi-2) is a Transformer model with 2.7 billion parameters. Performance on benchmarks for common sense, language understanding, and logical reasoning is nearly state-of-the-art among models with less than 13 billion parameters. Unlike typical Large Language Models (LLM), Phi-2 has not been fine-tuned through reinforcement learning from human feedback.""") | |
| else: | |
| prefix_post_processing_dict[t_opt] = os.environ["POST_PROCESSING_JSON_GEMMA"] | |
| st.write(""" | |
| Google Gemma-7b (https://huggingface.co/google/gemma-7b) is a Large Language Models (LLM) with 8.54 billion parameters. As per the https://storage.googleapis.com/deepmind-media/gemma/gemma-report.pdf, Gemma-7b performs well in language understanding, reasoning, and safety tasks. This model is one of state of the art open models built based on similar technologies that were used to create Google's Gemini models.""") | |
| st.markdown("---") | |
| # prefix_post_processing = os.environ["POST_PROCESSING_JSON"] | |
| st.header('Evaluation dataset') | |
| st.write(other_info_dict['data_description']) | |
| # %% | |
| st.header("Prompts") | |
| st.write("For each data point in the evaluation dataset and every prompt template described below, we create a prompt for LLM by adding the context and the question to the below prompt template, while following the same structure of the prompt template.") | |
| prompt_options_dict = { | |
| 'Prompt option 0 (with typos and grammatical errors, two shot prompting)': 'prompt_option_0.json', | |
| 'Prompt option 1 (Zero shot prompting)': 'prompt_option_1.json', | |
| 'Prompt option 2 (Prompt option 2 with two shot prompting)': 'prompt_option_2.json', | |
| 'Prompt option 3 (Prompt option 0 with minor fixes)': 'prompt_option_3.json' | |
| } | |
| prompt_options_list = list(prompt_options_dict.keys()) | |
| options = st.multiselect( | |
| 'Select one or more prompts:', | |
| prompt_options_list, | |
| [prompt_options_list[0]]) | |
| # st.write('Selected prompts:') | |
| # for t_opt in options: | |
| # st.write(t_opt) | |
| prompt_options_nums_list = [] | |
| st.markdown("---") | |
| for t_opt in options: | |
| st.write(t_opt) | |
| prompt_option = int(prompt_options_dict[t_opt].split('_')[-1].split('.')[0]) | |
| with open(f'prompt_{prompt_option}.txt', "r") as file: | |
| file_contents = file.read() | |
| # st.write(file_contents) | |
| st.text_area("Prompt template:", value=file_contents, height=100) | |
| st.markdown("---") | |
| prompt_options_nums_list += [prompt_option] | |
| result_processor_obj_dict = {} | |
| result_file_dict = {} | |
| data_dicts_dict = {} | |
| main_options = [] | |
| for model_option in model_select_options: | |
| for t_result_file in options: | |
| result_file = prefix_post_processing_dict[model_option] + prompt_options_dict[t_result_file] | |
| prompt_option = int(prompt_options_dict[t_result_file].split('_')[-1].split('.')[0]) | |
| with urllib.request.urlopen(result_file) as url: | |
| data_dict = json.load(url) | |
| result_processor_obj_dict[model_option + ' with ' + t_result_file] = ResultsProcessor( | |
| prompt_option=prompt_option, | |
| result_file=result_file, | |
| data_dict= data_dict | |
| ) | |
| data_dicts_dict[model_option + ' with ' + t_result_file] = data_dict | |
| result_file_dict[model_option + ' with ' + t_result_file] = result_file | |
| main_options += [model_option + ' with ' + t_result_file] | |
| print(result_processor_obj_dict) | |
| options = main_options | |
| # t_result_file = st.selectbox( | |
| # 'Select the prompt:', | |
| # list(prompt_options_dict.keys())) | |
| # result_file = prefix_post_processing + prompt_options_dict[t_result_file] | |
| # prompt_option = int(prompt_options_dict[t_result_file].split('_')[-1].split('.')[0]) | |
| # with urllib.request.urlopen(result_file) as url: | |
| # data_dict = json.load(url) | |
| # # File uploader | |
| # with open(f'prompt_{prompt_option}.txt', "r") as file: | |
| # file_contents = file.read() | |
| # # st.write(file_contents) | |
| # st.text_area("Prompt template:", value=file_contents, height=300) | |
| st.write("The answer to the question is obtained by post-processing the output of the LLM, wherein any additional content starting from the first 'Context: ' is disregarded.") | |
| st.write("In the case that the LLM answers <NO ANSWER>, the output is set to an empty string.") | |
| # 'Context: ' + context + '\n\n' + 'Question: ' + t_question + '\n\n' + 'Answer:' | |
| # %% | |
| st.header('Performance metric') | |
| st.write("""The performance metric used is an estimation of the percentage of correctly answered questions, i.e. where correctly answered questions mean that the output of the model coincides with one of the ground truth answers. The performance metric can also be interpreted as the probability that the model correctly answers a question. The performance of the model is evaluated with the exact match accuracy metric (see compute_exact function in SQuAD2.0 official evaluation script [here](https://worksheets.codalab.org/rest/bundles/0x6b567e1cf2e041ec80d7098f031c5c9e/contents/blob/)), taking values in [0,1], where 0 is worst (model always wrong), and 1 is best (model always correct). It is the number of correctly answered questions divided by the number of data points. An answer is considered to be correctly answered (by the model), if the predicted answer after normalization (text is converted to lowercase, and punctuation, articles and extra whitespace are removed) matches exactly with any of the normalized ground truth answers. In the case of unanswerable questions, the empty string is considered to be the only ground truth answer.""") | |
| with st.container(): | |
| overall_performance_list = [] | |
| for t_opt in options: | |
| overall_performance_list += [result_processor_obj_dict[t_opt].get_overall_performance()] | |
| t_perf_overall_df = pd.DataFrame({ | |
| 'Prompt' : options, | |
| 'Overall performance': overall_performance_list | |
| }) | |
| st.dataframe(t_perf_overall_df.set_index(t_perf_overall_df.columns[0])) | |
| perf_dict = t_perf_overall_df['Overall performance'].describe().round(2).to_dict() | |
| if len(options) > 1: | |
| st.subheader('Statistics of performance metrics across selected prompts') | |
| st.write('The statistics of overall performance across selected prompts is provided below.') | |
| temp_stat_df = pd.DataFrame({ | |
| 'Statistic': ['Mean', 'Standard deviation', 'Minimum', 'Maximum'], | |
| 'Value': [perf_dict['mean'], perf_dict['std'], perf_dict['min'], perf_dict['max']] | |
| }) | |
| st.dataframe(temp_stat_df.set_index(temp_stat_df.columns[0])) | |
| # st.write('Mean ', perf_dict['mean']) | |
| # st.write('Standard deviation: ', perf_dict['std']) | |
| # st.write('Minimum ', perf_dict['min']) | |
| # st.write('Maximum ', perf_dict['max']) | |
| # %% | |
| st.header("Bias ratios") | |
| st.write('Bias ratio is defined as the ratio of the lowest performance to the highest performance among categories that have sufficient data (with more than 50 data points) for a characteristic. The following table shows the bias ratio for each of the considered characteristics.') | |
| processed_t_opt_dict = {} | |
| ch_df = None | |
| for t_opt in options: | |
| processed_t_opt = t_opt.split('(')[0].strip() | |
| processed_t_opt_dict[t_opt] = processed_t_opt | |
| if ch_df is None: | |
| ch_df = result_processor_obj_dict[t_opt].get_bias_ratios_df() | |
| ch_df[processed_t_opt] = ch_df['Bias ratio'].values | |
| # ch_df.rename(columns={'Bias ratio': t_opt}) | |
| else: | |
| t_ch_df = result_processor_obj_dict[t_opt].get_bias_ratios_df() | |
| assert (ch_df['Characteristic'].values == t_ch_df['Characteristic'].values).all() | |
| ch_df[processed_t_opt] = t_ch_df['Bias ratio'].values | |
| ch_df.drop(columns=['Bias ratio'], axis=1, inplace=True) | |
| ch_df.set_index(ch_df.columns[0], inplace=True) | |
| # ch_df = result_processor_obj_dict[t_result_file].get_bias_ratios_df() | |
| if len(options) == 1: | |
| with st.container(): | |
| st.dataframe(ch_df) # Todo: MAX MIN HIGHLIGHT | |
| else: | |
| with st.container(): | |
| highlighted_df = ch_df.style.highlight_max(color='lightgreen', axis=1).highlight_min(color='lightcoral', axis=1) | |
| st.dataframe(highlighted_df) | |
| means_list = [] | |
| stds_list= [] | |
| mins_list = [] | |
| maxs_list = [] | |
| for i in ch_df.index: | |
| selected_columns = list(processed_t_opt_dict.keys()) # Example: Columns 'column1' and 'column3' | |
| # Create statistics for the selected rows across selected columns | |
| statistics = ch_df.loc[i].describe().loc[['mean', 'std', 'min', 'max']].to_dict() | |
| means_list += [statistics['mean']] | |
| stds_list += [statistics['std']] | |
| mins_list += [statistics['min']] | |
| maxs_list += [statistics['max']] | |
| t_ch_df = pd.DataFrame({ | |
| 'Mean': means_list, | |
| 'Standard deviation' : stds_list, | |
| 'Minimum': mins_list, | |
| 'Maximum': maxs_list | |
| }) | |
| t_ch_df.index = ch_df.index | |
| if len(options) > 1: | |
| with st.container(): | |
| st.write('For each characteristic, the statistics of bias ratios across selected prompt options is provided below.') | |
| st.dataframe(t_ch_df) | |
| # %% | |
| st.header("Robustness") | |
| st.write(f"""We evaluate the robustness of the LLM by assessing the variation in performance when perturbations are introduced to the content outside of the prompt template. The following plot shows the performance across different levels of perturbation within a perturbation family that consists of a series of perturbation methods. We consider the following perturbation families. | |
| - ProbTypos: {other_info_dict['ProbTypos_description']} | |
| - MaxTypo: {other_info_dict['MaxTypo_description']} | |
| """) | |
| # st.write(f"ProbTypos: {other_info_dict['ProbTypos_description']}") | |
| # st.write(f"MaxTypo: {other_info_dict['MaxTypo_description']}") | |
| t_pert_df_global_merged = [] | |
| for t_opt in options: | |
| st.write('Prompt used : ', t_opt) | |
| t_pert_df_global = result_processor_obj_dict[t_opt].get_global_perturbers_df() | |
| t_pert_df_global['Prompt option'] = processed_t_opt_dict[t_opt] | |
| t_pert_df_global['Perturbation family - Prompt'] = t_pert_df_global['Perturbation family'] + ' - ' + processed_t_opt_dict[t_opt] | |
| t_pert_fig = px.line(t_pert_df_global, x="Levels", y="Performance", color='Perturbation family') | |
| t_pert_fig.update_xaxes(tickmode='linear', dtick=1) | |
| t_pert_df_global_merged += [t_pert_df_global] | |
| st.plotly_chart(t_pert_fig, theme="streamlit", use_container_width=True) | |
| st.write('---') | |
| st.write('The following plot illustrates the variation in performance across perturbation families and prompts.') | |
| t_pert_df_global_merged_df = pd.concat(t_pert_df_global_merged, axis=0) | |
| t_pert_df_global_merged_df_2 = t_pert_df_global_merged_df[['Prompt option', 'Perturbation family','Levels', 'Performance']] | |
| t_pert_df_global_merged_df_2.set_index(t_pert_df_global_merged_df_2.columns[0]) | |
| # st.dataframe(t_pert_df_global_merged_df_2) | |
| temp_merged_fig = px.line(t_pert_df_global_merged_df, x="Levels", y="Performance", color='Perturbation family - Prompt') | |
| temp_merged_fig.update_xaxes(tickmode='linear', dtick=1) | |
| st.plotly_chart(temp_merged_fig, theme="streamlit", use_container_width=True) | |
| # %% | |
| st.header("Characteristic results") | |
| embedder_categories = data_dict['Embedder categories'] | |
| option = st.selectbox( | |
| 'Select characteristic:', | |
| sorted(list(embedder_categories.keys()))) | |
| st.write('The following are the categories:') | |
| st.write(', '.join(embedder_categories[option])) | |
| if 'Length' in option: | |
| st.write("Note: Here, length denotes the number of characters. ") | |
| if 'gender' in option: | |
| st.write(other_info_dict['gender_categories_text']) | |
| if 'ethnicity' in option: | |
| st.write(other_info_dict['ethnicity_categories_text']) | |
| t_df = result_processor_obj_dict[options[0]].get_data_distribution(option) | |
| fig = px.bar(t_df, x='Category', y='Number of points') | |
| st.plotly_chart(fig, theme="streamlit", use_container_width=True) | |
| st.markdown("---") | |
| st.write("The performance metric for each category is shown across the selected prompts.") | |
| count = 0 | |
| t_fair_dfs_list = [] | |
| for t_opt in options: | |
| t_fair_df = result_processor_obj_dict[t_opt].get_fairness_confidence_interval_df(option) | |
| t_fair_df['Prompt'] = processed_t_opt_dict[t_opt] | |
| t_fair_dfs_list += [t_fair_df] | |
| count +=1 | |
| merged_t_fair_df = pd.concat(t_fair_dfs_list, axis=0) | |
| fig_fair = px.scatter(merged_t_fair_df, x='Category', y='Estimate', color='Prompt', symbol='Prompt') | |
| # fig_fair = None | |
| fig_fair.update_layout(yaxis_title="Performance in %") | |
| st.plotly_chart(fig_fair, theme="streamlit", use_container_width=True) | |
| st.markdown("---") | |
| st.write("The performance metric is shown together with 95% confidence intervals for each category, across the selected prompts.") | |
| temp_options = st.multiselect( | |
| 'Choose from your pre-selected prompts:', | |
| options, | |
| [options[0]]) | |
| t_fair_dfs_list_map = {} | |
| count = 0 | |
| t_fair_dfs_list = [] | |
| for t_opt in temp_options: | |
| t_fair_df = result_processor_obj_dict[t_opt].get_fairness_confidence_interval_df(option) | |
| t_fair_df['Prompt'] = processed_t_opt_dict[t_opt] | |
| t_fair_dfs_list_map[t_opt] = count | |
| t_fair_dfs_list += [t_fair_df] | |
| count +=1 | |
| merged_t_fair_df = pd.concat(t_fair_dfs_list, axis=0) | |
| fig_fair = px.scatter(merged_t_fair_df, x='Category', y='Estimate', error_y='Diff upper', error_y_minus='Diff lower', color='Prompt', symbol='Prompt') | |
| fig_fair.update_layout(yaxis_title="Performance in %") | |
| st.plotly_chart(fig_fair, theme="streamlit", use_container_width=True) | |
| st.markdown("---") | |
| st.write('The following plots show the normalized average performance for each category of a characteristic, for each level of perturbation, starting with no perturbation. Each curve represents the normalized average performance on a category, by which we mean that we divide the average performance at every level of perturbation by the average performance without perturbation. ') | |
| temp_options_2 = st.multiselect( | |
| 'Choose from your pre-selected prompts:', | |
| options, | |
| [options[0]], key='Performance Robustness') | |
| st.markdown("---") | |
| for t_opt in temp_options_2: | |
| results_pert_rob_dict = result_processor_obj_dict[t_opt].get_performance_robustness(option) | |
| merged_dfs_list = results_pert_rob_dict['merged_dfs_list'] | |
| t_pert_df_global_temps_list = results_pert_rob_dict['t_pert_df_global_temps_list'] | |
| family_names_list = results_pert_rob_dict['family_names_list'] | |
| for merged_df, t_pert_df_global_temp, family_name in zip(merged_dfs_list, t_pert_df_global_temps_list, family_names_list): | |
| title_name = 'Perturbation family: ' + family_name + '\n\n Prompt : ' + t_opt | |
| t_pert_fig = px.line(merged_df, x="Levels", y="normalized performance", color='category') | |
| t_pert_fig.update_layout(yaxis_title="Normalized performance") | |
| t_pert_fig.add_trace(px.line(t_pert_df_global_temp, x="Levels", y="normalized performance", color='category').data[0]) | |
| t_pert_fig.update_xaxes(tickmode='linear') | |
| # t_pert_fig.update_layout(title=title_name) | |
| # st.write(f'The following plot illustrates the normalized performance of the model across different categories for the perturbation family: {family_name}.') | |
| st.write(title_name) | |
| st.plotly_chart(t_pert_fig, theme="streamlit", use_container_width=True) | |
| st.markdown("---") | |