Boubou78000 commited on
Commit
0a88a11
1 Parent(s): 2c3f7c0
Files changed (1) hide show
  1. app.py +16 -20
app.py CHANGED
@@ -5,6 +5,7 @@ from datasets import load_dataset
5
  from transformers import AutoTokenizer
6
 
7
  def ReturnTokens(dataset_name, tokenizer_name="openai-community/gpt2", split="train"):
 
8
  # Initialize tokenizer
9
  tokenizer = AutoTokenizer.from_pretrained(tokenizer_name)
10
 
@@ -34,26 +35,21 @@ with gr.Blocks(title="Dataset token counter") as app:
34
  tokenizer = gr.Textbox(label="Tokenizer", elem_id="tokenizer", info="", placeholder="openai-community/gpt2", value="openai-community/gpt2")
35
  split = gr.Textbox(label="Split (default: train)", elem_id="split", info="", placeholder="train", value="train")
36
  tokens = gr.Label(label="Tokens", elem_id="tokens")
37
- prompt.submit().success(
38
- ReturnTokens,
39
- inputs=[prompt,tokenizer,split],
40
- outputs=[tokens]
41
- )
42
 
43
- gr.on(
44
- triggers=[
45
- prompt.submit,
46
- tokenizer.submit,
47
- split.submit,
48
- ],
49
- fn=ReturnTokens,
50
- inputs=[
51
- prompt,
52
- tokenizer,
53
- split
54
- ],
55
- outputs=[tokens],
56
- api_name="run",
57
- )
58
 
59
  app.launch()
 
5
  from transformers import AutoTokenizer
6
 
7
  def ReturnTokens(dataset_name, tokenizer_name="openai-community/gpt2", split="train"):
8
+
9
  # Initialize tokenizer
10
  tokenizer = AutoTokenizer.from_pretrained(tokenizer_name)
11
 
 
35
  tokenizer = gr.Textbox(label="Tokenizer", elem_id="tokenizer", info="", placeholder="openai-community/gpt2", value="openai-community/gpt2")
36
  split = gr.Textbox(label="Split (default: train)", elem_id="split", info="", placeholder="train", value="train")
37
  tokens = gr.Label(label="Tokens", elem_id="tokens")
 
 
 
 
 
38
 
39
+ gr.on(
40
+ triggers=[
41
+ prompt.submit,
42
+ tokenizer.submit,
43
+ split.submit,
44
+ ],
45
+ fn=ReturnTokens,
46
+ inputs=[
47
+ prompt,
48
+ tokenizer,
49
+ split
50
+ ],
51
+ outputs=[tokens],
52
+ api_name="run",
53
+ )
54
 
55
  app.launch()