cxeep commited on
Commit
2af0b9f
1 Parent(s): ad99185

Create app.py

Browse files
Files changed (1) hide show
  1. app.py +28 -0
app.py ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # https://huggingface.co/spaces/JacobLinCool/tiktoken-calculator/blob/main/app.py
2
+ import gradio as gr
3
+ import tiktoken
4
+
5
+ encodings = tiktoken.list_encoding_names()
6
+ encodings.reverse()
7
+
8
+
9
+ def calc(input: str, encoding: str) -> int:
10
+ tokens = tiktoken.get_encoding(encoding).encode(input)
11
+ return len(tokens)
12
+
13
+
14
+ gr.Interface(
15
+ calc,
16
+ inputs=[
17
+ gr.Text(label="Input", lines=6, placeholder="Enter text here..."),
18
+ gr.Dropdown(
19
+ label="Encoding",
20
+ choices=encodings,
21
+ value="cl100k_base",
22
+ info="The encoding to use. (GPT-3.5 and GPT-4 use cl100k_base as their encoding.)",
23
+ ),
24
+ ],
25
+ outputs=gr.Number(label="Output"),
26
+ title="Tiktoken Calculator",
27
+ allow_flagging="never",
28
+ ).launch()