Update app.py

app.py

@@ -1,22 +1,33 @@
 import gradio as gr
+from random import random
 
-# Placeholder function for loading the tokenizer
-def load_tokenizer(tokenizer_choice):
-    # Placeholder implementation - load your tokenizer here based on the choice
-    return None
+# Assuming `aranizer` is a library that provides these tokenizer classes or functions (pseudo-code for demonstration only)
+# Please implement actual imports and tokenizer initialization logic
+def get_tokenizer(tokenizer_choice):
+    # Placeholder - Replace with actual tokenizer loading logic
+    tokenizer_map = {
+        "aranizer_bpe32k": None,  # Replace None with actual tokenizer, e.g., aranizer_bpe32k.get_tokenizer()
+        # Add other tokenizers here
+    }
+    return tokenizer_map.get(tokenizer_choice, None)
 
-def tokenize_and_encode(text, tokenizer_choice):
-    tokenizer = load_tokenizer(tokenizer_choice)
+def tokenize_and_encode_and_embed(text, tokenizer_choice):
+    tokenizer = get_tokenizer(tokenizer_choice)
     if tokenizer:
+        # Example methods. Replace with actual methods from your tokenizer
         tokens = tokenizer.tokenize(text)
         encoded_output = tokenizer.encode(text, add_special_tokens=True)
         decoded_text = tokenizer.decode(encoded_output)
-        return " ".join(tokens), str(encoded_output), decoded_text
+        
+        # Example embedding (replace with actual embedding generation from your model)
+        embeddings = [random() for _ in range(10)]  # Example 10-dimensional embedding vector
+        
+        return " ".join(tokens), str(encoded_output), decoded_text, embeddings
     else:
-        return "Tokenizer not loaded correctly", "", ""
+        return "Tokenizer not loaded correctly", "", "", []
 
 demo = gr.Interface(
-    fn=tokenize_and_encode,
+    fn=tokenize_and_encode_and_embed,
     inputs=[
         gr.Textbox(lines=5, label="النص العربي"),
         gr.Dropdown(choices=["aranizer_bpe32k", "aranizer_bpe50k", "aranizer_bpe64k", "aranizer_bpe86k", "aranizer_sp32k", "aranizer_sp50k", "aranizer_sp64k", "aranizer_sp86k"], label="اختر المحلل اللفظي")
@@ -24,10 +35,11 @@ demo = gr.Interface(
     outputs=[
         gr.Text(label="Tokens"),
         gr.Text(label="Encoded Output"),
-        gr.Text(label="Decoded Text")
+        gr.Text(label="Decoded Text"),
+        gr.Text(label="Embeddings (Example Vector)")
     ],
-    title="مقارنة المحللات اللفظية للنص العربي",
-    description="حدد نوع المحلل اللفظي وأدخل نصًا لرؤية النتائج."
+    title="مقارنة المحللات اللفظية وعمليات التضمين للنص العربي",
+    description="حدد نوع المحلل اللفظي وأدخل نصًا لرؤية النتائج ومتجه التضمين.",
 )
 
 demo.launch()