HeshamHaroon commited on
Commit
4aae5f1
1 Parent(s): c8d4230

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +4 -4
app.py CHANGED
@@ -4,12 +4,12 @@ import aranizer
4
  from transformers import AutoTokenizer
5
  import codecs
6
 
7
- # Loading tokenizer instances from Transformers.
8
  gpt_13b_tokenizer = AutoTokenizer.from_pretrained("FreedomIntelligence/AceGPT-13B")
9
  gpt_7b_tokenizer = AutoTokenizer.from_pretrained("FreedomIntelligence/AceGPT-7B")
10
  jais_13b_tokenizer = AutoTokenizer.from_pretrained("inception-mbzuai/jais-13b")
11
 
12
- # Assuming the existence of get_tokenizer() method for aranizer models in your setup.
13
  tokenizers = {
14
  "aranizer_bpe50k": lambda: aranizer.aranizer_bpe50k.get_tokenizer(),
15
  "aranizer_bpe64k": lambda: aranizer.aranizer_bpe64k.get_tokenizer(),
@@ -23,7 +23,7 @@ tokenizers = {
23
  "inception-mbzuai/jais-13b": lambda: jais_13b_tokenizer,
24
  }
25
 
26
- # Define tokenizer options for dropdown menu.
27
  tokenizer_options = list(tokenizers.keys())
28
 
29
  def compare_tokenizers(tokenizer_name, text):
@@ -44,7 +44,7 @@ def compare_tokenizers(tokenizer_name, text):
44
 
45
  inputs_component = [
46
  gr.Dropdown(choices=tokenizer_options, label="Select Tokenizer"),
47
- gr.Textbox(lines=2, placeholder="Enter Arabic text here...", label="Input Text", default="مثال بالعربية")
48
  ]
49
 
50
  outputs_component = gr.Dataframe(
 
4
  from transformers import AutoTokenizer
5
  import codecs
6
 
7
+ # Loading tokenizer instances from Transformers
8
  gpt_13b_tokenizer = AutoTokenizer.from_pretrained("FreedomIntelligence/AceGPT-13B")
9
  gpt_7b_tokenizer = AutoTokenizer.from_pretrained("FreedomIntelligence/AceGPT-7B")
10
  jais_13b_tokenizer = AutoTokenizer.from_pretrained("inception-mbzuai/jais-13b")
11
 
12
+ # Assuming the existence of get_tokenizer() method for aranizer models in your setup
13
  tokenizers = {
14
  "aranizer_bpe50k": lambda: aranizer.aranizer_bpe50k.get_tokenizer(),
15
  "aranizer_bpe64k": lambda: aranizer.aranizer_bpe64k.get_tokenizer(),
 
23
  "inception-mbzuai/jais-13b": lambda: jais_13b_tokenizer,
24
  }
25
 
26
+ # Define tokenizer options for dropdown menu
27
  tokenizer_options = list(tokenizers.keys())
28
 
29
  def compare_tokenizers(tokenizer_name, text):
 
44
 
45
  inputs_component = [
46
  gr.Dropdown(choices=tokenizer_options, label="Select Tokenizer"),
47
+ gr.Textbox(lines=2, placeholder="Enter Arabic text here...", label="Input Text") # Removed `default` parameter
48
  ]
49
 
50
  outputs_component = gr.Dataframe(