Create app.py

app.py

@@ -0,0 +1,15 @@
+from transformers import AutoTokenizer
+import gradio as gr
+
+
+gpt2_tokenizer = AutoTokenizer.from_pretrained("gpt2")
+gptj_tokenizer = AutoTokenizer.from_pretrained("EleutherAI/gpt-j-6b")
+gpt_neox_tokenizer = AutoTokenizer.from_pretrained("EleutherAI/gpt-neox-20b")
+llama_tokenizer = AutoTokenizer.from_pretrained("hf-internal-testing/llama-tokenizer")
+
+def tokenize(input_text):
+    tokens = tokenizer(input_text)["input_ids"]
+    return f"Number of tokens: {len(tokens)}"
+
+iface = gr.Interface(fn=tokenize, inputs=gr.inputs.Textbox(lines=7), outputs="text")
+iface.launch()