File size: 357 Bytes
a0047d0
0dc240c
 
e2c8253
a0047d0
0dc240c
 
e2c8253
0dc240c
 
a0047d0
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
from transformers import AutoTokenizer
import gradio as gr


tokenizer = AutoTokenizer.from_pretrained("kiranr/gpt2-tokenizer")

def tokenize(input_text):
    tokens = tokenizer(input_text)["input_ids"]
    return f"Number of tokens: {len(tokens)}"


demo = gr.Interface(
    fn=tokenize,
    inputs=gr.Textbox(lines=7),
    outputs="text",
)
demo.launch()