Update app.py

app.py

@@ -42,6 +42,8 @@ def compare_tokenizers(tokenizer_name, text):
         # AraNizer tokenizers
         tokenizer = tokenizers[tokenizer_name]()
         tokens = tokenizer.tokenize(text)
+        for token in tokens:
+            print(token.encode('utf-8').decode('utf-8'))
         encoded_output = tokenizer.encode(text, add_special_tokens=True)
         decoded_text = tokenizer.decode(encoded_output)