Created app

app.py

@@ -2,18 +2,21 @@ import token
 import tokenize
 import gradio as gr
 from datasets import load_dataset
-from tokenizers import Tokenizer
+from transformers import AutoTokenizer
 
 def ReturnTokens(dataset, tokenizer="openai-community/gpt2", split="train"):
     global tokens_
-    tokenizer=Tokenizer.from_pretrained(tokenizer)
-    dataset=load_dataset(dataset)
+    tokenizer=AutoTokenizer.from_pretrained(tokenizer)
+    dataset=load_dataset(dataset, split=split)
     tokens_=0
     def CountTokens(Example):
         global tokens_
-        for i in Example.values():
-            tokens_+=len(Tokenizer.encode(i))
-    dataset.map(CountTokens)
+        print(Example)
+        for k,i in enumerate(Example):
+            tokens_+=len(tokenizer.tokenize(i))
+    categories=[i for i in dataset[0].keys()]
+    for cat in categories:
+        CountTokens(dataset[cat])
     return tokens_
 
 with gr.Blocks(title="Dataset token counter") as app:
@@ -30,4 +33,20 @@ with gr.Blocks(title="Dataset token counter") as app:
             outputs=[tokens]
         )
 
-app.launch(share=True)
+        gr.on(
+            triggers=[
+                prompt.submit,
+                tokenizer.submit,
+                split.submit,
+            ],
+            fn=ReturnTokens,
+            inputs=[
+                prompt,
+                tokenizer,
+                split
+            ],
+            outputs=[tokens],
+            api_name="run",
+        )
+
+app.launch()