Spaces:
Runtime error
Runtime error
Commit
•
778be61
1
Parent(s):
71c7fc4
Update app.py
Browse files
app.py
CHANGED
@@ -70,12 +70,16 @@ def compare_tokenizers(tokenizer_name, text):
|
|
70 |
decoded_text = tokenizer.decode(encoded_output, skip_special_tokens=True)
|
71 |
|
72 |
# Prepare the results to be displayed in HTML format
|
|
|
|
|
|
|
|
|
73 |
results_html = f"""
|
74 |
-
<div>
|
75 |
<h3>Tokenizer: {tokenizer_name}</h3>
|
76 |
-
<p><strong>Tokens:</strong> {
|
77 |
-
<p><strong>Encoded:</strong> {
|
78 |
-
<p><strong>Decoded:</strong> {
|
79 |
</div>
|
80 |
"""
|
81 |
return results_html
|
|
|
70 |
decoded_text = tokenizer.decode(encoded_output, skip_special_tokens=True)
|
71 |
|
72 |
# Prepare the results to be displayed in HTML format
|
73 |
+
tokens_html = "".join([f"<span style='background-color:#D3D3D3; padding:2px; margin:2px; border-radius:5px;'>{token}</span>" for token in tokens])
|
74 |
+
encoded_html = "".join([f"<span style='background-color:#ADD8E6; padding:2px; margin:2px; border-radius:5px;'>{token}</span>" for token in encoded_output])
|
75 |
+
decoded_html = f"<div style='background-color:#90EE90; padding:10px; border-radius:5px;'>{decoded_text}</div>"
|
76 |
+
|
77 |
results_html = f"""
|
78 |
+
<div style='font-family: Arial, sans-serif;'>
|
79 |
<h3>Tokenizer: {tokenizer_name}</h3>
|
80 |
+
<p><strong>Tokens:</strong> {tokens_html}</p>
|
81 |
+
<p><strong>Encoded:</strong> {encoded_html}</p>
|
82 |
+
<p><strong>Decoded:</strong> {decoded_html}</p>
|
83 |
</div>
|
84 |
"""
|
85 |
return results_html
|