Make the code to use precalculated tokens from the dataset

app.py

@@ -1,20 +1,12 @@
 import streamlit as st
-from collections import defaultdict
-import tqdm
-import transformers
-from transformers import AutoTokenizer
 import pandas as pd
-import matplotlib.pyplot as plt
-import seaborn as sns
-import numpy as np
-import plotly.figure_factory as ff
 import plotly.express as px
 from plotly.subplots import make_subplots
 import plotly.graph_objects as go
-import random, glob
+import random
 
 
-@st.cache_data
+@st.cache
 def load_data():
     return pd.read_csv("dataset.csv")
 
@@ -22,7 +14,6 @@ def load_data():
 def reload_example_text_data(language):
     random_id = random.choice(val_data["id"])
     tempdf = val_data[val_data["id"] == random_id]
-    tempdf = tempdf[["iso", "text", *selected_tokenizers]]
     tempdf = tempdf[tempdf["iso"] == language]
     tempdf.set_index("iso", inplace=True)
     tempdf.columns = ["Text"] + [f"Num Tokens ({t})" for t in selected_tokenizers]
@@ -50,11 +41,8 @@ tokenizer_names_to_test = [
 
 with st.sidebar:
     st.header("Comparing Tokenizers")
-    link = "This project compares the tokenization length for different tokenizers. Some tokenizers may result in significantly more tokens than others for the same text."
+    link = "This project compares the tokenization length for different tokenizers."
     st.markdown(link)
-
-    st.header("Data Visualization")
-    st.subheader("Tokenizers")
     selected_tokenizers = st.multiselect(
         "Select tokenizers",
         options=tokenizer_names_to_test,
@@ -63,29 +51,25 @@ with st.sidebar:
         label_visibility="collapsed",
     )
 
-    st.subheader("Data")
-    with st.spinner("Loading dataset..."):
-        val_data = load_data()
-    st.success(f"Data loaded: {len(val_data)}")
+    for tokenizer_name in selected_tokenizers:
+        if tokenizer_name == "openai/gpt4":
+            link = "Tokenized using [tiktoken](https://github.com/openai/tiktoken)"
+        else:
+            url = f"https://huggingface.co/{tokenizer_name}"
+            link = f"Tokenizer is available [on the HuggingFace hub]({url})"
+        st.markdown(link, unsafe_allow_html=True)
 
-    with st.expander("Data Source"):
-        st.write(
-            "The data in this figure is the validation set of the [Amazon Massive](https://huggingface.co/datasets/AmazonScience/massive/viewer/af-ZA/validation) dataset, which consists of 2033 short sentences and phrases translated into 51 different languages. Learn more about the dataset from [Amazon's blog post](https://www.amazon.science/blog/amazon-releases-51-language-dataset-for-language-understanding)"
-        )
+    val_data = load_data()
+    st.success(f"Data loaded: {len(val_data)}")
 
-    st.subheader("Language")
-    language_options = sorted(val_data.lang.unique())
-    default_language_index = (
-        language_options.index("English") if "English" in language_options else 0
-    )
+    language_options = sorted(val_data["lang"].unique())
     selected_language = st.selectbox(
         "Select language",
         options=language_options,
-        index=default_language_index,
+        index=language_options.index("English") if "English" in language_options else 0,
         label_visibility="collapsed",
     )
 
-    st.subheader("Figure")
     selected_figure = st.radio(
         "Select figure type",
         options=["Boxplot", "Histogram", "Scatterplot"],
@@ -94,18 +78,13 @@ with st.sidebar:
     )
 
 st.header("Example Text")
-with st.spinner("Loading example text..."):
-    reload_example_text_data(selected_language)
+reload_example_text_data(selected_language)
 st.table(st.session_state.examplesdf)
 st.button("Reload", on_click=reload_example_text_data, args=(selected_language,))
 
-tokenizer_to_num_tokens = defaultdict(list)
-for _, row in tqdm.tqdm(val_data.iterrows(), total=len(val_data)):
-    text = row["text"]
-    for tokenizer_name in selected_tokenizers:
-        tokenizer = AutoTokenizer.from_pretrained(tokenizer_name)
-        num_tokens = len(tokenizer(text)["input_ids"])
-        tokenizer_to_num_tokens[tokenizer_name].append(num_tokens)
+tokenizer_to_num_tokens = {
+    name: val_data[name].tolist() for name in selected_tokenizers
+}
 
 if selected_figure == "Boxplot":
     fig = go.Figure()
@@ -113,11 +92,7 @@ if selected_figure == "Boxplot":
         fig.add_trace(
             go.Box(y=tokenizer_to_num_tokens[tokenizer_name], name=tokenizer_name)
         )
-    fig.update_layout(
-        title=f"Distribution of Number of Tokens for Selected Tokenizers",
-        xaxis_title="Tokenizer",
-        yaxis_title="Number of Tokens",
-    )
+    fig.update_layout(title="Distribution of Number of Tokens for Selected Tokenizers")
     st.plotly_chart(fig)
 elif selected_figure == "Histogram":
     fig = make_subplots(
@@ -138,14 +113,8 @@ elif selected_figure == "Histogram":
     st.plotly_chart(fig)
 elif selected_figure == "Scatterplot":
     df = pd.DataFrame(tokenizer_to_num_tokens)
-    fig = px.scatter_matrix(
-        df,
-        dimensions=selected_tokenizers,
-        color_discrete_sequence=px.colors.qualitative.Plotly,
-    )
+    fig = px.scatter_matrix(df, dimensions=selected_tokenizers)
     fig.update_layout(
-        title=f"Scatterplot Matrix of Number of Tokens for Selected Tokenizers",
-        width=800,
-        height=800,
+        title="Scatterplot Matrix of Number of Tokens for Selected Tokenizers"
     )
     st.plotly_chart(fig)