Idea

app.py

@@ -1,18 +1,35 @@
+import token
+import tokenize
 import gradio as gr
-from transformers import pipeline
+from datasets import load_dataset
+from tokenizers import Tokenizer
 
-pipeline = pipeline(task="image-classification", model="julien-c/hotdog-not-hotdog")
+from StreetFlash.Farm import Get
 
-def predict(input_img):
-    predictions = pipeline(input_img)
-    return input_img, {p["label"]: p["score"] for p in predictions} 
+def ReturnTokens(dataset, tokenizer="openai-community/gpt2", split="train"):
+    global tokens_
+    tokenizer=Tokenizer.from_pretrained(tokenizer)
+    dataset=load_dataset(dataset)
+    tokens_=0
+    def CountTokens(Example):
+        global tokens_
+        for i in Example.values():
+            tokens_+=len(Tokenizer.encode(i))
+    dataset.map(CountTokens)
+    return tokens_
 
-gradio_app = gr.Interface(
-    predict,
-    inputs=gr.Image(label="Select hot dog candidate", sources=['upload', 'webcam'], type="pil"),
-    outputs=[gr.Image(label="Processed Image"), gr.Label(label="Result", num_top_classes=2)],
-    title="Hot Dog? Or Not?",
-)
+with gr.Blocks(title="Dataset token counter") as app:
+    gr.Markdown("# Token Counter")
 
-if __name__ == "__main__":
-    gradio_app.launch()
+    with gr.Row():
+        prompt = gr.Textbox(label="Dataset", elem_id="dataset", info="", placeholder="")
+        tokenizer = gr.Textbox(label="Tokenizer", elem_id="tokenizer", info="", placeholder="openai-community/gpt2", value="openai-community/gpt2")
+        split = gr.Textbox(label="Split (default: train)", elem_id="split", info="", placeholder="train", value="train")
+        tokens = gr.Label(label="Tokens", elem_id="tokens", info="")
+        prompt.submit().success(
+            ReturnTokens,
+            inputs=[prompt,tokenizer,split],
+            outputs=[tokens]
+        )
+
+app.launch()