Spaces:
Runtime error
Runtime error
| ### LIBRARIES ### | |
| # # Data | |
| from matplotlib.pyplot import legend | |
| import numpy as np | |
| import pandas as pd | |
| import torch | |
| import json | |
| from tqdm import tqdm | |
| from math import floor | |
| from datasets import load_dataset | |
| from collections import defaultdict | |
| from transformers import AutoTokenizer | |
| # Analysis | |
| # from gensim.models.doc2vec import Doc2Vec | |
| # from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score | |
| # import nltk | |
| # nltk.download('punkt') #make sure that punkt is downloaded | |
| # App & Visualization | |
| import streamlit as st | |
| import st_aggrid | |
| import altair as alt | |
| import plotly.graph_objects as go | |
| from streamlit_vega_lite import altair_component | |
| # utils | |
| from random import sample | |
| # from PIL import Image | |
| def down_samp(embedding): | |
| """Down sample a data frame for altiar visualization """ | |
| # total number of positive and negative sentiments in the class | |
| #embedding = embedding.groupby('slice').apply(lambda x: x.sample(frac=0.3)) | |
| total_size = embedding.groupby(['slice','label'], as_index=False).count() | |
| user_data = 0 | |
| # if 'Your Sentences' in str(total_size['slice']): | |
| # tmp = embedding.groupby(['slice'], as_index=False).count() | |
| # val = int(tmp[tmp['slice'] == "Your Sentences"]['source']) | |
| # user_data = val | |
| max_sample = total_size.groupby('slice').max()['content'] | |
| # # down sample to meeting altair's max values | |
| # # but keep the proportional representation of groups | |
| down_samp = 1/(sum(max_sample.astype(float))/(1000-user_data)) | |
| max_samp = max_sample.apply(lambda x: floor(x*down_samp)).astype(int).to_dict() | |
| max_samp['Your Sentences'] = user_data | |
| # # sample down for each group in the data frame | |
| embedding = embedding.groupby('slice').apply(lambda x: x.sample(n=max_samp.get(x.name))).reset_index(drop=True) | |
| # # order the embedding | |
| return(embedding) | |
| def data_comparison(df): | |
| # set up a dropdown select bindinf | |
| # input_dropdown = alt.binding_select(options=['Negative Sentiment','Positive Sentiment']) | |
| selection = alt.selection_multi(fields=['slice','label']) | |
| color = alt.condition(alt.datum.slice == 'high-loss', alt.value("orange"), alt.value("steelblue")) | |
| # color = alt.condition(selection, | |
| # alt.Color('slice:Q', legend=None), | |
| # # scale = alt.Scale(domain = pop_domain,range=color_range)), | |
| # alt.value('lightgray')) | |
| opacity = alt.condition(selection, alt.value(0.7), alt.value(0.25)) | |
| # basic chart | |
| scatter = alt.Chart(df).mark_point(size=100, filled=True).encode( | |
| x=alt.X('x', axis=None), | |
| y=alt.Y('y', axis=None), | |
| color=color, | |
| shape=alt.Shape('label', scale=alt.Scale(range=['circle', 'diamond'])), | |
| tooltip=['slice','content','label','pred'], | |
| opacity=opacity | |
| ).properties( | |
| width=1500, | |
| height=1000 | |
| ).interactive() | |
| legend = alt.Chart(df).mark_point().encode( | |
| y=alt.Y('slice:N', axis=alt.Axis(orient='right'), title=""), | |
| x=alt.X("label"), | |
| shape=alt.Shape('label', scale=alt.Scale( | |
| range=['circle', 'diamond']), legend=None), | |
| color=color | |
| ).add_selection( | |
| selection | |
| ) | |
| layered = scatter |legend | |
| layered = layered.configure_axis( | |
| grid=False | |
| ).configure_view( | |
| strokeOpacity=0 | |
| ) | |
| return layered | |
| def quant_panel(embedding_df): | |
| """ Quantitative Panel Layout""" | |
| all_metrics = {} | |
| # st.warning("**Data Comparison**") | |
| # with st.expander("how to read this chart:"): | |
| # st.markdown("* each **point** is a single sentence") | |
| # st.markdown("* the **position** of each dot is determined mathematically based upon an analysis of the words in a sentence. The **closer** two points on the visualization the **more similar** the sentences are. The **further apart ** two points on the visualization the **more different** the sentences are") | |
| # st.markdown( | |
| # " * the **shape** of each point reflects whether it a positive (diamond) or negative sentiment (circle)") | |
| # st.markdown("* the **color** of each point is the ") | |
| st.altair_chart(data_comparison(down_samp(embedding_df))) | |
| def frequent_tokens(data, tokenizer, loss_quantile=0.95, top_k=200, smoothing=0.005): | |
| unique_tokens = [] | |
| tokens = [] | |
| for row in tqdm(data['content']): | |
| tokenized = tokenizer(row,padding=True, return_tensors='pt') | |
| tokens.append(tokenized['input_ids'].flatten()) | |
| unique_tokens.append(torch.unique(tokenized['input_ids'])) | |
| losses = data['loss'].astype(float) | |
| high_loss = losses.quantile(loss_quantile) | |
| loss_weights = (losses > high_loss) | |
| loss_weights = loss_weights / loss_weights.sum() | |
| token_frequencies = defaultdict(float) | |
| token_frequencies_error = defaultdict(float) | |
| weights_uniform = np.full_like(loss_weights, 1 / len(loss_weights)) | |
| num_examples = len(data) | |
| for i in tqdm(range(num_examples)): | |
| for token in unique_tokens[i]: | |
| token_frequencies[token.item()] += weights_uniform[i] | |
| token_frequencies_error[token.item()] += loss_weights[i] | |
| token_lrs = {k: (smoothing+token_frequencies_error[k]) / (smoothing+token_frequencies[k]) for k in token_frequencies} | |
| tokens_sorted = list(map(lambda x: x[0], sorted(token_lrs.items(), key=lambda x: x[1])[::-1])) | |
| top_tokens = [] | |
| for i, (token) in enumerate(tokens_sorted[:top_k]): | |
| top_tokens.append(['%10s' % (tokenizer.decode(token)), '%.4f' % (token_frequencies[token]), '%.4f' % ( | |
| token_frequencies_error[token]), '%4.2f' % (token_lrs[token])]) | |
| return pd.DataFrame(top_tokens, columns=['Token', 'Freq', 'Freq error slice', 'lrs']) | |
| def get_data(spotlight, emb): | |
| preds = spotlight.outputs.numpy() | |
| losses = spotlight.losses.numpy() | |
| embeddings = pd.DataFrame(emb, columns=['x', 'y']) | |
| num_examples = len(losses) | |
| # dataset_labels = [dataset[i]['label'] for i in range(num_examples)] | |
| return pd.concat([pd.DataFrame(np.transpose(np.vstack([dataset[:num_examples]['content'], | |
| dataset[:num_examples]['label'], preds, losses])), columns=['content', 'label', 'pred', 'loss']), embeddings], axis=1) | |
| def topic_distribution(weights, smoothing=0.01): | |
| topic_frequencies = defaultdict(float) | |
| topic_frequencies_spotlight = defaultdict(float) | |
| weights_uniform = np.full_like(weights, 1 / len(weights)) | |
| num_examples = len(weights) | |
| for i in range(num_examples): | |
| example = dataset[i] | |
| category = example['title'] | |
| topic_frequencies[category] += weights_uniform[i] | |
| topic_frequencies_spotlight[category] += weights[i] | |
| topic_ratios = {c: (smoothing + topic_frequencies_spotlight[c]) / ( | |
| smoothing + topic_frequencies[c]) for c in topic_frequencies} | |
| categories_sorted = map(lambda x: x[0], sorted( | |
| topic_ratios.items(), key=lambda x: x[1], reverse=True)) | |
| topic_distr = [] | |
| for category in categories_sorted: | |
| topic_distr.append(['%.3f' % topic_frequencies[category], '%.3f' % | |
| topic_frequencies_spotlight[category], '%.2f' % topic_ratios[category], '%s' % category]) | |
| return pd.DataFrame(topic_distr, columns=['Overall frequency', 'Error frequency', 'Ratio', 'Category']) | |
| # for category in categories_sorted: | |
| # return(topic_frequencies[category], topic_frequencies_spotlight[category], topic_ratios[category], category) | |
| if __name__ == "__main__": | |
| ### STREAMLIT APP CONGFIG ### | |
| st.set_page_config(layout="wide", page_title="Error Slice Analysis") | |
| lcol, rcol = st.columns([3, 2]) | |
| # ******* loading the mode and the data | |
| dataset = st.sidebar.selectbox( | |
| "Dataset", | |
| ["amazon_polarity", "squad", "movielens", "waterbirds"], | |
| index=0 | |
| ) | |
| tokenizer = AutoTokenizer.from_pretrained( | |
| "distilbert-base-uncased-finetuned-sst-2-english") | |
| model = st.sidebar.selectbox( | |
| "Model", | |
| ["distilbert-base-uncased-finetuned-sst-2-english", | |
| "distilbert-base-uncased-finetuned-sst-2-english"], | |
| index=0 | |
| ) | |
| loss_quantile = st.sidebar.selectbox( | |
| "Loss Quantile", | |
| [0.98, 0.95, 0.9, 0.8, 0.75], | |
| index = 1 | |
| ) | |
| ### LOAD DATA AND SESSION VARIABLES ### | |
| data_df = pd.read_parquet('amazon_polarity.test.parquet') | |
| embedding_umap = data_df[['x','y']] | |
| if "user_data" not in st.session_state: | |
| st.session_state["user_data"] = data_df | |
| if "selected_slice" not in st.session_state: | |
| st.session_state["selected_slice"] = None | |
| if "embedding" not in st.session_state: | |
| st.session_state["embedding"] = embedding_umap | |
| with lcol: | |
| st.title('Error Slices') | |
| dataframe = data_df[['content', 'label', 'pred', 'loss']].sort_values( | |
| by=['loss'], ascending=False) | |
| table_html = dataframe.to_html( | |
| columns=['content', 'label', 'pred', 'loss'], max_rows=100) | |
| # table_html = table_html.replace("<th>", '<th align="left">') # left-align the headers | |
| st.write(dataframe) | |
| # st_aggrid.AgGrid(dataframe) | |
| # table_html = dataframe.to_html(columns=['content', 'label', 'pred', 'loss'], max_rows=100) | |
| # table_html = table_html.replace("<th>", '<th align="left">') # left-align the headers | |
| # st.write(table_html) | |
| with rcol: | |
| st.title('Word Distribution in Error Slice') | |
| commontokens = frequent_tokens(data_df, tokenizer, loss_quantile=loss_quantile) | |
| st.write(commontokens) | |
| data_df['loss'] = data_df['loss'].astype(float) | |
| losses = data_df['loss'] | |
| high_loss = losses.quantile(loss_quantile) | |
| data_df['slice'] = 'high-loss' | |
| data_df['slice'] = data_df['slice'].where(data_df['loss'] > high_loss, 'low-loss') | |
| quant_panel(data_df) | |