xzuyn commited on
Commit
e46a0b5
1 Parent(s): d8bd459

add box which shows encoded tokens, also add labels

Browse files
Files changed (1) hide show
  1. app.py +4 -3
app.py CHANGED
@@ -5,7 +5,8 @@ sp = SentencePieceProcessor(model_file="tokenizer.model")
5
 
6
  def tokenize(input_text):
7
  tokens = sp.EncodeAsIds(input_text)
8
- return f"Number of tokens: {len(tokens)}"
 
9
 
10
- iface = gr.Interface(fn=tokenize, inputs=gr.inputs.Textbox(lines=7), outputs="text")
11
- iface.launch()
 
5
 
6
  def tokenize(input_text):
7
  tokens = sp.EncodeAsIds(input_text)
8
+ decoded_tokens = sp.DecodeIds(tokens)
9
+ return len(tokens), tokens
10
 
11
+ iface = gr.Interface(fn=tokenize, inputs=gr.inputs.Textbox(lines=7, label="Input Text"), outputs=[gr.outputs.Textbox(label="Token Count"), gr.outputs.Textbox(label="Tokens")])
12
+ iface.launch()