Spaces:
Runtime error
Runtime error
Update app.py
Browse files
app.py
CHANGED
@@ -4,36 +4,24 @@ import tiktoken
|
|
4 |
encodings = tiktoken.list_encoding_names()
|
5 |
encodings.reverse()
|
6 |
|
7 |
-
|
8 |
-
|
9 |
-
tokens =
|
10 |
return len(tokens)
|
11 |
-
|
12 |
-
value1 = gr.Textbox(lines=6, label="Input", placeholder="Input the text...")
|
13 |
-
value2 = gr.Dropdown(
|
14 |
-
label="Encoding",
|
15 |
-
choices=encodings,
|
16 |
-
default="cl100k_base",
|
17 |
-
description="The encoding to use. (GPT-3.5 and GPT-4 use cl100k_base as their encoding.)"
|
18 |
-
)
|
19 |
-
value3 = gr.Textbox(lines=3, label="Output", placeholder="Output...")
|
20 |
-
|
21 |
-
examples = [
|
22 |
-
["The only way to do great work is to love what you do. - Steve Jobs"],
|
23 |
-
["In the end, we will remember not the words of our enemies, but the silence of our friends. - Martin Luther King Jr."],
|
24 |
-
["Success is not final, failure is not fatal: It is the courage to continue that counts. - Winston Churchill"],
|
25 |
-
["The greatest glory in living lies not in never falling, but in rising every time we fall. - Nelson Mandela"],
|
26 |
-
["The best and most beautiful things in the world cannot be seen or even touched - they must be felt with the heart. - Helen Keller"]
|
27 |
-
]
|
28 |
|
29 |
-
demo = gr.Interface(
|
30 |
-
fn=function,
|
31 |
-
inputs=[value1, value2],
|
32 |
-
outputs=value3,
|
33 |
-
title="ChatGPT Token Calculator",
|
34 |
-
examples=examples,
|
35 |
-
description="Calculate the number of tokens in a text string.o",
|
36 |
-
allow_flagging="never"
|
37 |
-
)
|
38 |
|
39 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
4 |
encodings = tiktoken.list_encoding_names()
|
5 |
encodings.reverse()
|
6 |
|
7 |
+
|
8 |
+
def calc(input: str, encoding: str) -> int:
|
9 |
+
tokens = tiktoken.get_encoding(encoding).encode(input)
|
10 |
return len(tokens)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
11 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
12 |
|
13 |
+
gr.Interface(
|
14 |
+
calc,
|
15 |
+
inputs=[
|
16 |
+
gr.Text(label="Input", lines=6, placeholder="Enter text here..."),
|
17 |
+
gr.Dropdown(
|
18 |
+
label="Encoding",
|
19 |
+
choices=encodings,
|
20 |
+
value="cl100k_base",
|
21 |
+
info="The encoding to use. (GPT-3.5 and GPT-4 use cl100k_base as their encoding.)",
|
22 |
+
),
|
23 |
+
],
|
24 |
+
outputs=gr.Number(label="Output"),
|
25 |
+
title="Tiktoken Calculator",
|
26 |
+
allow_flagging="never",
|
27 |
+
).launch()
|