Update app.py
Browse files
app.py
CHANGED
@@ -15,9 +15,10 @@ def get_tokenizer(name):
|
|
15 |
def examine(tokenizer, tok_name, inp):
|
16 |
enc = tokenizer.encode(inp)
|
17 |
reconstructed_inp = tokenizer.decode(enc)
|
|
|
18 |
st.header(f"{tok_name}")
|
19 |
st.subheader(f"#tokens = {len(enc)}")
|
20 |
-
st.text(f"Exact match = {
|
21 |
st.code(reconstructed_inp)
|
22 |
st.write(enc)
|
23 |
|
|
|
15 |
def examine(tokenizer, tok_name, inp):
|
16 |
enc = tokenizer.encode(inp)
|
17 |
reconstructed_inp = tokenizer.decode(enc)
|
18 |
+
reconstructed_inp2 = tokenizer.decode(enc, skip_special_tokens=True)
|
19 |
st.header(f"{tok_name}")
|
20 |
st.subheader(f"#tokens = {len(enc)}")
|
21 |
+
st.text(f"Exact match = {reconstructed_inp2 == inp}")
|
22 |
st.code(reconstructed_inp)
|
23 |
st.write(enc)
|
24 |
|