chenyeyuan
commited on
Commit
•
aa92a8a
1
Parent(s):
570a00c
tokenizer
Browse files- app.py +37 -0
- neox_tokenizer.json +0 -0
- requirements.txt +3 -0
app.py
ADDED
@@ -0,0 +1,37 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import streamlit as st
|
2 |
+
from transformers import AutoTokenizer, PreTrainedTokenizerFast
|
3 |
+
from streamlit_ace import st_ace
|
4 |
+
|
5 |
+
|
6 |
+
@st.cache(allow_output_mutation=True)
|
7 |
+
def get_tokenizer(name):
|
8 |
+
return (
|
9 |
+
PreTrainedTokenizerFast(tokenizer_file="neox_tokenizer.json")
|
10 |
+
if name == "gpt-neox"
|
11 |
+
else AutoTokenizer.from_pretrained(name)
|
12 |
+
)
|
13 |
+
|
14 |
+
|
15 |
+
def examine(tokenizer, tok_name, inp):
|
16 |
+
enc = tokenizer.encode(inp)
|
17 |
+
reconstructed_inp = tokenizer.decode(enc)
|
18 |
+
st.header(f"{tok_name}")
|
19 |
+
st.subheader(f"#tokens = {len(enc)}")
|
20 |
+
st.text(f"Exact match = {reconstructed_inp == inp}")
|
21 |
+
st.code(reconstructed_inp)
|
22 |
+
st.write(enc)
|
23 |
+
|
24 |
+
|
25 |
+
tokenizer_name_1 = st.text_input("Tokenizer 1 to examine", "gpt-neox")
|
26 |
+
tokenizer_name_2 = st.text_input("Tokenizer 2 to examine", "t5-small")
|
27 |
+
tokenizer1 = get_tokenizer(tokenizer_name_1)
|
28 |
+
tokenizer2 = get_tokenizer(tokenizer_name_2)
|
29 |
+
inp = st_ace(
|
30 |
+
"Input text",
|
31 |
+
"Find {eq}\\dfrac{\\partial f}{\\partial x} \\text{ and } \\dfrac{\\partial f}{\\partial y}{/eq} for {eq}f(x,y) = 13(8x - 7y+3)^5{/eq}.",
|
32 |
+
)
|
33 |
+
col0, col1 = st.columns(2)
|
34 |
+
with col0:
|
35 |
+
examine(tokenizer1, tokenizer_name_1, inp)
|
36 |
+
with col1:
|
37 |
+
examine(tokenizer2, tokenizer_name_2, inp)
|
neox_tokenizer.json
ADDED
The diff for this file is too large to render.
See raw diff
|
|
requirements.txt
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
transformers
|
2 |
+
streamlit==1.8.0
|
3 |
+
streamlit_ace==0.1.1
|