Update app.py

app.py

@@ -40,8 +40,8 @@ def compare_tokenizers(tokenizer_name, text):
         tokenizer = tokenizers[tokenizer_name]()
         tokens = tokenizer.tokenize(text)
         tokens_arabic = [token.encode('utf-8').decode('utf-8') for token in tokens]
-        encoded_output = tokenizer.encode(text, add_special_tokens=True, return_tensors="pt")
-        decoded_text = tokenizer.decode(encoded_output[0], skip_special_tokens=True)
+        encoded_output = tokenizer.encode(text, add_special_tokens=True)
+        decoded_text = tokenizer.decode(encoded_output, skip_special_tokens=True)
     else:
         # AraNizer tokenizers
         tokenizer = tokenizers[tokenizer_name]()
@@ -51,7 +51,7 @@ def compare_tokenizers(tokenizer_name, text):
         tokens_arabic = [token.encode('utf-8').decode('utf-8') for token in tokens]
     
     # Prepare the results to be displayed
-    results = [(tokenizer_name, tokens_arabic, encoded_output.tolist(), decoded_text)]
+    results = [(tokenizer_name, tokens_arabic, encoded_output, decoded_text)]
     return results
 
 # Define the Gradio interface components with a dropdown for model selection