Add more preset tokenizers

app.py

@@ -9,27 +9,35 @@ from datasets import load_dataset
 from PIL import Image
 from transformers import AutoTokenizer
 
+
 tokenizers = {
     "bert": "google-bert/bert-base-uncased",
+    "blenderbot": "facebook/blenderbot-3B",
     "bloom": "bigscience/bloom-560m",
-    "gemma": "fxmarty/tiny-random-GemmaForCausalLM",
+    "bloomz": "bigscience/bloomz-7b1",
     "chatglm3": "THUDM/chatglm3-6b",
     "falcon": "tiiuae/falcon-7b",
+    "gemma": "fxmarty/tiny-random-GemmaForCausalLM",
     "gpt-neox": "EleutherAI/gpt-neox-20b",
     "llama": "TinyLlama/TinyLlama-1.1B-Chat-v0.6",
     "magicoder": "ise-uiuc/Magicoder-S-DS-6.7B",
     "mistral": "mistralai/Mistral-7B-v0.1",
+    "mpt": "mosaicml/mpt-7b",
     "opt": "facebook/opt-2.7b",
     "phi-2": "microsoft/phi-2",
     "pythia": "EleutherAI/pythia-1.4b-deduped",
-    "roberta": "FacebookAI/roberta-base",
     "qwen": "Qwen/Qwen1.5-7B-Chat",
+    "redpajama": "togethercomputer/RedPajama-INCITE-Chat-3B-v1",
+    "roberta": "FacebookAI/roberta-base",
     "starcoder": "bigcode/starcoder2-7b",
     "t5": "google-t5/t5-base",
+    "vicuna": "lmsys/vicuna-7b-v1.5",
+    "zephyr": "HuggingFaceH4/zephyr-7b-beta",
 }
 
 tokenizers = list(tokenizers.values())
 
+
 def plot_histogram(data):
     plt.hist(data)
     plt.title("Histogram of number of tokens per dataset item")
@@ -98,7 +106,7 @@ demo = gr.Interface(
         ["mistralai/Mistral-7B-v0.1", "wikitext", "wikitext-2-v1", "validation", "text"],
         ["mistralai/Mistral-7B-v0.1", "zeroshot/twitter-financial-news-sentiment", "", "validation", "text"],
     ],
-    cache_examples=True
+    cache_examples=True,
 )
 
 demo.launch()