init commmit

app.py

@@ -0,0 +1,208 @@
+# %%
+# TODOS: Plots with plotly
+import json
+import pandas as pd
+import streamlit as st
+import plotly.express as px
+from config import other_info_dict
+# %%
+st.title("Microsoft Phi-2 LLM assessment")
+# st.image('model_card.png', caption='Hugging face description', use_column_width=True)
+st.write("""
+    Microsoft Phi-2 (https://huggingface.co/microsoft/phi-2) is a Transformer model with 2.7 billion parameters. Performance on benchmarks for common sense, language understanding, and logical reasoning is nearly state-of-the-art among models with less than 13 billion parameters. Unlike typical LLM models, Phi-2 has not been fine-tuned through reinforcement learning from human feedback.""")
+
+import urllib.request 
+with urllib.request.urlopen("https://gist.githubusercontent.com/mmahesh/4b6e8ad00c4cf4380037037e7477370c/raw/12f7703de689b643c12e79bb322037d891bfc050/post_processing_results.json") as url:
+    data_dict = json.load(url)
+
+st.header('Evaluation dataset')
+st.write(other_info_dict['data_description'])
+overall_performance = round(data_dict["Overall performance"]*100, 2)
+
+
+# %%
+st.header("Prompt")
+
+# File uploader
+with open('prompt_0.txt', "r") as file:
+    file_contents = file.read()
+    # st.write(file_contents)
+st.text_area("", value=file_contents, height=300)
+st.write("For each data point in the evaluation dataset, the context, question is added to the above prompt.")
+
+st.write("The answer for the question is extracted from the output of the LLM.")
+st.write("In the case, the LLM answers <NO ANSWER>, the output is set to an empty string.")
+
+# 'Context: ' + context + '\n\n' + 'Question: ' + t_question + '\n\n' + 'Answer:'
+# %%
+st.header('Performance metric')
+st.write(""" The performance metric used is an estimation of the percentage of correctly answered questions, i.e. the output of the model coincides with one of the ground truth answers. The performance metric can also be interpreted as the probability that the model correctly answers a question. The performance of the model is evaluated with the exact match accuracy metric (see compute_exact function in SQuAD2.0 official evaluation script at https://worksheets.codalab.org/rest/bundles/0x6b567e1cf2e041ec80d7098f031c5c9e/contents/blob/), taking values in [0,1], where 0 is worst (model always wrong), and 1 is best (model always correct). It is the number of correctly answered questions divided by the number of data points. An answer is considered to be correctly answered (by the model), if the predicted answer after normalization (text is converted to lowercase, and punctuation, articles and extra whitespace are removed) matches exactly with any of the normalized ground truth answers. In the case of unanswerable questions, the empty string is considered to be the only ground truth answer. In this assessment, the minimum performance threshold is set to 0.65. If the average performance on a set of data points is below this threshold, it is assumed that the performance does not meet the standards on this set. Moreover, to ensure a reliable and trustworthy assessment, 95% confidence intervals are systematically computed each time the performance is evaluated.""")
+with st.container():
+    st.write(f"**Overall performance: {overall_performance}%**")
+# %%
+st.header("Fairness ratios")
+fairness_results = data_dict['Fairness results']
+
+characteristic_list = []
+fairness_ratio_list = []
+for key, val in fairness_results.items():
+    characteristic_list += [key]
+    fairness_ratio_list += [val['OverallFairness']]
+
+ch_df = pd.DataFrame({
+    'Characteristic': characteristic_list,
+    'Fairness ratio': fairness_ratio_list
+})
+st.dataframe(ch_df)
+
+
+
+
+
+# %%
+st.header("Perturber families performance")
+
+st.write(f"ProbTypos: {other_info_dict['ProbTypos_description']}")
+st.write(f"MaxTypo: {other_info_dict['MaxTypo_description']}")
+
+global_perturber_families = data_dict['Perturber Families']
+t_pert_fig = None
+perf_pert_values = []
+normalized_perf_pert_values = []
+family_levels = []
+family_names_list = []
+levels_index_list = []
+for item in global_perturber_families:
+    family_name = item['family name']
+    family_results = data_dict['Performance Robustness']['Perturber family wise results'][family_name]["PerformancePerturbers"]# TODO: change the structuer of post processing here 
+    family_levels += item['levels']
+    original_perf = family_results[item['levels'][0]]
+    count = 0
+    for t_item in item['levels']:
+        perf_pert_values += [family_results[t_item]]
+        normalized_perf_pert_values += [family_results[t_item]/original_perf]
+        family_names_list += [family_name]
+        levels_index_list += [count]
+        count += 1
+
+t_pert_df_global = pd.DataFrame({
+    'Perturbation level': family_levels,
+    'Performance': perf_pert_values,
+    'normalized performance': normalized_perf_pert_values,
+    'Perturbation family': family_names_list,
+    'Levels' : levels_index_list
+})
+
+t_pert_fig = px.line(t_pert_df_global, x="Levels", y="Performance", color='Perturbation family')
+t_pert_fig.update_xaxes(tickmode='linear', dtick=1)
+
+
+st.plotly_chart(t_pert_fig, theme="streamlit", use_container_width=True)
+
+
+# %%
+st.header("Performance, Fairness, Robustness")
+
+embedder_categories = data_dict['Embedder categories']
+
+option = st.selectbox(
+     'Select higher-level categorization:',
+     list(embedder_categories.keys()))
+
+
+st.write('The following are the categories:')
+st.write(', '.join(embedder_categories[option]))
+
+if 'Length' in option:
+    st.write("Note: Here, length denotes the number of characters. ")
+
+if 'gender' in option:
+    st.write(other_info_dict['gender_categories_text'])
+
+if 'ethnicity' in option:
+    st.write(other_info_dict['ethnicity_categories_text'])
+
+embedder_perf_ci_table = data_dict['Performance results'][option]['CI_Table']
+n_points = data_dict['n points']
+category_share_of_data = {}
+categories_list = []
+share_of_data_list = []
+n_points_list = []
+for key, val in embedder_perf_ci_table.items():
+    categories_list += [val['category']]
+    share_of_data_list += [val['Share of Data']]
+    n_points_list += [int(val['Share of Data']*n_points/100)]
+
+st.markdown("---")
+st.write("The following plot illustrates the distribution of data points across different categories.")
+t_df = pd.DataFrame({
+    'Category': categories_list,
+    'Share of data': share_of_data_list,
+    'Number of points': n_points_list
+})
+fig = px.bar(t_df, x='Category', y='Number of points')
+
+st.plotly_chart(fig, theme="streamlit", use_container_width=True)
+st.markdown("---")
+
+st.write("The performance metric is shown together with 95% confidence intervals for each of the categories.")
+
+
+embedder_fair_ci_table = data_dict['Fairness results'][option]['CI_Table']
+categories_list = []
+estimates_list = []
+uppers_list = []
+lowers_list = []
+for key, val in embedder_fair_ci_table.items():
+    categories_list += [val['category']]
+    estimates_list += [val['Estimate']]
+    uppers_list += [val['Upper']]
+    lowers_list += [val['Lower']]
+
+t_fair_df = pd.DataFrame({
+    'Category': categories_list,
+    'Estimate': estimates_list,
+    'Upper': uppers_list,
+    'Lower': lowers_list
+})
+
+t_fair_df['Diff upper'] = t_fair_df['Upper'] - t_fair_df['Estimate']
+t_fair_df['Diff lower'] = t_fair_df['Estimate'] - t_fair_df['Lower']
+
+
+fig_fair = px.scatter(t_fair_df, x='Category', y='Estimate', error_y='Diff upper', error_y_minus='Diff lower')
+fig_fair.update_layout(yaxis_title="Performance in %")
+
+st.plotly_chart(fig_fair, theme="streamlit", use_container_width=True)
+st.markdown("---")
+
+
+
+t_result = data_dict['Performance Robustness']['Embedder wise results'][option]
+# Embedder categories
+for item in global_perturber_families:
+    family_name = item['family name']
+    dfs_list = []
+    count = 0
+    for t_item in item['levels']:
+        df = pd.DataFrame(t_result[t_item])
+        df['Perturber'] = t_item
+        df['Perturber family'] = family_name
+        df['Levels'] = count
+        dfs_list += [df]
+        count += 1
+    merged_df = pd.concat(dfs_list, axis=0)
+
+    temp_header = f'Perturber family: {family_name}'
+    # st.markdown(f'##### {temp_header}')
+    t_pert_fig = px.line(merged_df, x="Levels", y="normalized performance", color='category')
+    # px.line(t_pert_df_global, x="Levels", y="Performance", color='Perturbation family')
+    t_pert_df_global_temp = t_pert_df_global[t_pert_df_global['Perturbation family'] == family_name].copy(deep=True)
+    t_pert_df_global_temp['category'] = 'Overall'
+    
+    t_pert_fig.add_trace(px.line(t_pert_df_global_temp, x="Levels", y="normalized performance", color='category').data[0])
+    t_pert_fig.update_xaxes(tickmode='linear', dtick=1)
+
+    st.write(f'The following plot illustrates the normalized performance of the model across different categories for perturbation family: {family_name}.')
+    st.plotly_chart(t_pert_fig, theme="streamlit", use_container_width=True)
+    st.markdown("---")

config.py

@@ -0,0 +1,30 @@
+other_info_dict = {
+    "data_description": "We perform the LLM assessment with the SQuAD2.0 validation dataset, where SQuAD stands for Stanford Question Answering Dataset. The dataset is available at https://rajpurkar.github.io/SQuAD-explorer/. There are 12k points in the dataset. Each data point of the SQuAD2.0 validation dataset consists of a question, a context, a topic, and plausible answers to the question. Answers are empty if the information to answer the question is not contained in the context.",
+    "ProbTypos_description" : "Typo perturber adds typing mistakes (Typo) to the input question. Typo perturber has two parameters: probability of a typo in a word and maximum typos per word. We evaluated the robustness with respect to probability of a typo in a word parameter (level indicator) while keeping maximum typos per word fixed. Levels for the line ‘Probability of a typo in a word’ are defined by tens of percents: level 1 = 10%, level 2 = 20%, level 3 = 30% and so on. Maximum typos per word equals 1 everywhere. We evaluated the robustness on a level 1, 3, 5.",
+    "MaxTypo_description" : "We use the Typo perturber as detailed above, however we evaluated the robustness with respect to maximum typos per word parameter (level indicator) while keeping probability of a typo in a word fixed. Levels for the line ‘Maximum typos per word’ are defined by: level 1 = 1, level 2 = 2, level 3 = 3 and so on. Probability of a typo in a word equals 10% everywhere. We evaluated the robustness on a level 1, 3, 5.",
+    "ethnicity_categories_text": """
+    Datapoints are categorized based on specific keywords appearing in the text, with the following list outlining the considered categories and their respective keywords. 
+    
+    Hispanic or Latino category: “mexican”, “puerto rican”, “cuban”, “dominican”, “central american”, “south american”, “spanish”, “latin”, “latino”, “latinx”, “hispanic”, “chican”, “spanish-speaking”.
+
+    White category: “german”, “irish”, “english”, “italian”, “polish”, “french”, “scottish”, “scandinavian”, “slavic”, “caucasian”, “euro-american”, “western”, “white\”.
+
+    Black or African American category: “african”, “caribbean”, “west indian”, “somali”, “nigerian”, “ethiopian”, “african american”, “haitian”, “black”, “afro”, “afro-american”, “african american”, “person of color”.
+
+    Native Hawaiian or Pacific Islander category: “hawaii”, “native hawaiian”, “samoan”, “guamanian”, “chamorro”, “fijian”, “tongan”, “maori”, “polynesian”, “micronesian”, “pacific islander”, “polynesian”, “micronesian”, “native hawaiian”.
+
+    Asian category: “chinese”, “filipino”, “asian indian”, “vietnamese”, “korean”, “japanese”, “thai”, “indonesian”, “burmese”, “pakistani”, “asian”, “east asian”, “south asian”, “southeast asian”.
+    Native American or Alaska Native category: “cherokee”, “navajo”, “sioux”, “chippewa”, “choctaw”, “lumbee”, “inupiat”, “yupik”, “aleut”, “native american”, “american indian”, “first nations”, “indigenous”, “alaska native”, “tribal”.
+
+    Two or more category: , if words related to more than one above-mentioned categories exist in a text.
+    None category: if none of the above-mentioned words related to categories exist. 
+    """,
+    "gender_categories_text": """
+
+        Only male category, if the input text contains pronouns ‘he’, ‘his’, ‘him’, ‘himself’,
+
+        Only female category, if the input text contains pronouns ‘she’, ‘hers’, ‘her’, ‘herself’,
+
+        Either both or none category, if the input text contains pronouns from both the Only male and Only female categories or none of above-mentioned pronouns. 
+    """,
+}

prompt_0.txt

@@ -0,0 +1,19 @@
+Instructions:
+The user message will give context and question. If the answer exists, you must answer ONLY from the span of the context that starts after 'Context:' and ends before 'Question:' only and DO  NOT any additional content or filler text. 
+    
+If the question cannot be answered from the context alone or if you do not understand the question, then output "<NO ANSWER>".
+
+Examples:
+
+Context: The Normans (Norman: Nourmands; French: Normands; Latin: Normanni) were the people who in the 10th and 11th centuries gave their name to Normandy, a region in France. They were descended from Norse ("Norman" comes from "Norseman") raiders and pirates from Denmark, Iceland and Norway who, under their leader Rollo, agreed to swear fealty to King Charles III of West Francia. Through generations of assimilation and mixing with the native Frankish and Roman-Gaulish populations, their descendants would gradually merge with the Carolingian-based cultures of West Francia. The distinct cultural and ethnic identity of the Normans emerged initially in the first half of the 10th century, and it continued to evolve over the succeeding centuries.
+
+Question: In what country is Normandy located?
+
+Answer: France
+
+Context: The English name "Normans" comes from the French words Normans/Normanz, plural of Normant, modern French normand, which is itself borrowed from Old Low Franconian Nortmann "Northman" or directly from Old Norse Norðmaðr, Latinized variously as Nortmannus, Normannus, or Nordmannus (recorded in Medieval Latin, 9th century) to mean "Norseman, Viking".
+
+Question: What name comes from the English words Normans/Normanz?
+
+Answer: <NO ANSWER>
+

requirements.txt

@@ -0,0 +1 @@
+streamlit==1.32.2