Update app.py

app.py

@@ -218,8 +218,8 @@ for d in dataset:
     data_per_lang[full] = data_per_lang.get(code, []) + [d[k]]
 
 
-def get_results(tokenizer_name, base_lang, comp_lang):
-    tokenizer = AutoTokenizer.from_pretrained(tokenizer_name)
+def get_results(tokenizer_name, base_lang, comp_lang, HF_token=""):
+    tokenizer = AutoTokenizer.from_pretrained(tokenizer_name, token=HF_token if HF_token != "" else False)
     base_data = data_per_lang[base_lang]
     comp_data = data_per_lang[comp_lang]
 
@@ -251,8 +251,11 @@ def get_results(tokenizer_name, base_lang, comp_lang):
 
 with gr.Blocks() as demo:
     with gr.Column():
-        with gr.Row():
-            tokenizer = gr.Textbox(label="Tokenizer name", value="bert-base-cased")
+        with gr.Column():
+            with gr.Row():
+                tokenizer = gr.Textbox(label="Tokenizer name", value="bert-base-cased")
+            with gr.Row():
+                HF_token = gr.Textbox(label="your HF Token")
 
         with gr.Row():
             with gr.Column():
@@ -271,7 +274,7 @@ with gr.Blocks() as demo:
 
     btn.click(
             get_results,
-            inputs=[tokenizer, base_lang, comp_lang],
+            inputs=[tokenizer, base_lang, comp_lang, HF_token],
             outputs=[out_text],
             api_name=False,
         )