HeshamHaroon's picture
Create app.py
fcbfd45 verified
raw
history blame
1.34 kB
import gradio as gr
# Assuming we have a function to load a tokenizer by name (you would need to replace this with actual implementation)
def load_tokenizer(tokenizer_name):
if tokenizer_name == "aranizer_sp32k":
tokenizer = aranizer_sp32k.get_tokenizer()
# Add conditions for other tokenizers
return tokenizer
def tokenize_and_encode(text, tokenizer_choice):
tokenizer = load_tokenizer(tokenizer_choice)
tokens = tokenizer.tokenize(text)
encoded_output = tokenizer.encode(text, add_special_tokens=True)
decoded_text = tokenizer.decode(encoded_output)
return tokens, encoded_output, decoded_text
iface = gr.Interface(
fn=tokenize_and_encode,
inputs=[gr.inputs.Textbox(lines=5, label="النص العربي"), gr.inputs.Dropdown(choices=["aranizer_bpe32k", "aranizer_bpe50k", "aranizer_bpe64k", "aranizer_bpe86k", "aranizer_sp32k", "aranizer_sp50k", "aranizer_sp64k", "aranizer_sp86k"], label="اختر المحلل اللفظي")],
outputs=[gr.outputs.Textbox(label="Tokens"), gr.outputs.Textbox(label="Encoded Output"), gr.outputs.Textbox(label="Decoded Text")],
title="مقارنة المحللات اللفظية للنص العربي",
description="حدد نوع المحلل اللفظي وأدخل نصًا لرؤية النتائج.",
language="ar",
)
iface.launch()