Spaces:
Runtime error
Runtime error
HeshamHaroon
commited on
Commit
•
3f8c44e
1
Parent(s):
27c0e8b
Update app.py
Browse files
app.py
CHANGED
@@ -74,13 +74,19 @@ def compare_tokenizers(tokenizer_index, text):
|
|
74 |
tokens_display = [token.encode('utf-8').decode('utf-8') if isinstance(token, bytes) else token for token in tokens]
|
75 |
|
76 |
# Prepare the results to be displayed in HTML format
|
77 |
-
tokens_html = "".join([
|
78 |
-
|
79 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
80 |
|
81 |
results_html = f"""
|
82 |
<div style='font-family: Arial, sans-serif;'>
|
83 |
-
<h3 style='color: #
|
84 |
<p><strong>Tokens:</strong> {tokens_html}</p>
|
85 |
<p><strong>Encoded:</strong> {encoded_html}</p>
|
86 |
<p><strong>Decoded:</strong> {decoded_html}</p>
|
|
|
74 |
tokens_display = [token.encode('utf-8').decode('utf-8') if isinstance(token, bytes) else token for token in tokens]
|
75 |
|
76 |
# Prepare the results to be displayed in HTML format
|
77 |
+
tokens_html = "".join([
|
78 |
+
f"<span style='background-color:#eeeeee; color: #333333; padding:4px; margin:2px; border-radius:3px; border:1px solid #cccccc;'>{token}</span>"
|
79 |
+
for token in tokens_display
|
80 |
+
])
|
81 |
+
encoded_html = "".join([
|
82 |
+
f"<span style='background-color:#e0e0e0; color: #000000; padding:4px; margin:2px; border-radius:3px; border:1px solid #aaaaaa;'>{token}</span>"
|
83 |
+
for token in encoded_output
|
84 |
+
])
|
85 |
+
decoded_html = f"<div style='background-color:#f5f5f5; color: #444444; padding:10px; border-radius:3px; border:1px solid #999999;'>{decoded_text}</div>"
|
86 |
|
87 |
results_html = f"""
|
88 |
<div style='font-family: Arial, sans-serif;'>
|
89 |
+
<h3 style='color: #2e7d32;'>Tokenizer: {tokenizer_name}</h3>
|
90 |
<p><strong>Tokens:</strong> {tokens_html}</p>
|
91 |
<p><strong>Encoded:</strong> {encoded_html}</p>
|
92 |
<p><strong>Decoded:</strong> {decoded_html}</p>
|