Update app.py

app.py

@@ -3,15 +3,36 @@ import gradio as gr
 
 
 def tokenize(input_text):
-    llama_tokens = llama_tokenizer(input_text, add_special_tokens=True)["input_ids"]
-    mistral_tokens = mistral_tokenizer(input_text, add_special_tokens=True)["input_ids"]
-    gpt2_tokens = gpt2_tokenizer(input_text, add_special_tokens=True)["input_ids"]
-    gpt_neox_tokens = gpt_neox_tokenizer(input_text, add_special_tokens=True)["input_ids"]
-    falcon_tokens = falcon_tokenizer(input_text, add_special_tokens=True)["input_ids"]
-    phi2_tokens = phi2_tokenizer(input_text, add_special_tokens=True)["input_ids"]
-    t5_tokens = t5_tokenizer(input_text, add_special_tokens=True)["input_ids"]
-
-    return f"LLaMa: {len(llama_tokens)}\nMistral: {len(mistral_tokens)}\nGPT-2/GPT-J: {len(gpt2_tokens)}\nGPT-NeoX: {len(gpt_neox_tokens)}\nFalcon: {len(falcon_tokens)}\nPhi-2: {len(phi2_tokens)}\nT5: {len(t5_tokens)}"
+    llama_tokens = len(
+        llama_tokenizer(input_text, add_special_tokens=True)["input_ids"]
+    )
+    mistral_tokens = len(
+        mistral_tokenizer(input_text, add_special_tokens=True)["input_ids"]
+    )
+    gpt2_tokens = len(gpt2_tokenizer(input_text, add_special_tokens=True)["input_ids"])
+    gpt_neox_tokens = len(
+        gpt_neox_tokenizer(input_text, add_special_tokens=True)["input_ids"]
+    )
+    falcon_tokens = len(
+        falcon_tokenizer(input_text, add_special_tokens=True)["input_ids"]
+    )
+    phi2_tokens = len(phi2_tokenizer(input_text, add_special_tokens=True)["input_ids"])
+    t5_tokens = len(t5_tokenizer(input_text, add_special_tokens=True)["input_ids"])
+
+    results = {
+        "LLaMa": llama_tokens,
+        "Mistral": mistral_tokens,
+        "GPT-2/GPT-J": gpt2_tokens,
+        "GPT-NeoX": gpt_neox_tokens,
+        "Falcon": falcon_tokens,
+        "Phi-2": phi2_tokens,
+        "T5": t5_tokens,
+    }
+
+    # Sort the results in descending order based on token length
+    sorted_results = sorted(results.items(), key=lambda x: x[1], reverse=True)
+
+    return "\n".join([f"{model}: {tokens}" for model, tokens in sorted_results])
 
 
 if __name__ == "__main__":