HeshamHaroon commited on
Commit
00fb4c8
1 Parent(s): 91eabe4

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +1 -1
app.py CHANGED
@@ -70,7 +70,7 @@ def compare_tokenizers(tokenizer_name, text):
70
  decoded_text = tokenizer.decode(encoded_output, skip_special_tokens=True)
71
 
72
  # Ensure the tokens are properly decoded
73
- tokens_display = [token for token in tokens]
74
 
75
  # Prepare the results to be displayed in HTML format
76
  tokens_html = "".join([f"<span style='background-color:#D3D3D3; padding:2px; margin:2px; border-radius:5px;'>{token}</span>" for token in tokens_display])
 
70
  decoded_text = tokenizer.decode(encoded_output, skip_special_tokens=True)
71
 
72
  # Ensure the tokens are properly decoded
73
+ tokens_display = [token.encode('utf-8').decode('utf-8') if isinstance(token, bytes) else token for token in tokens]
74
 
75
  # Prepare the results to be displayed in HTML format
76
  tokens_html = "".join([f"<span style='background-color:#D3D3D3; padding:2px; margin:2px; border-radius:5px;'>{token}</span>" for token in tokens_display])