added multilingual_translation library

app.py

@@ -1,5 +1,8 @@
 from transformers import pipeline, set_seed
 from transformers import BioGptTokenizer, BioGptForCausalLM
+from multilingual_translation import translate
+from utils import lang_ids
+
 import gradio as gr
 
 model_list = [
@@ -7,18 +10,30 @@ model_list = [
     "microsoft/BioGPT-Large-PubMedQA"
 ]
 
+lang_list = list(lang_ids.keys())
+
+def translate_to_english(text, base_lang):
+    if base_lang == "en":
+        return text
+    else:
+        new_text = translate("facebook/m2m100_418M", text, base_lang, "en")
+        return new_text
+    
+
 def biogpt(
     prompt: str,
     model_id: str,
     max_length: int = 25,
-    num_return_sequences: int = 5
+    num_return_sequences: int = 5,
+    base_lang: str = "en"
 ):
     
+    en_prompt = translate_to_english(prompt, base_lang)
     model = BioGptForCausalLM.from_pretrained(model_id)
     tokenizer = BioGptTokenizer.from_pretrained(model_id)
     generator = pipeline('text-generation', model=model, tokenizer=tokenizer)
     set_seed(42)
-    output = generator(prompt, max_length=max_length, num_return_sequences=num_return_sequences, do_sample=True)
+    output = generator(en_prompt, max_length=max_length, num_return_sequences=num_return_sequences, do_sample=True)
     output_dict = {
         "1": output[0]['generated_text'],
         "2": output[1]['generated_text'],
@@ -33,13 +48,16 @@ inputs = [
     gr.inputs.Textbox(label="Prompt", lines=5, default="COVID-19 is"),
     gr.Dropdown(model_list, value="microsoft/biogpt", label="Model ID"),
     gr.inputs.Slider(5, 100, 25, default=25, label="Max Length"),
-    gr.inputs.Slider(1, 10, 5, default=5, label="Num Return Sequences")
+    gr.inputs.Slider(1, 10, 5, default=5, label="Num Return Sequences"),
+    gr.Dropdown(lang_list, value="en", label="Base Language")
     ]
 
 outputs = gr.outputs.Textbox(label="Output")
 
 examples = [
-    ["COVID-19 is", "microsoft/biogpt", 25, 5]
+    ["COVID-19 is", "microsoft/biogpt", 25, 5, "en"],
+    ["Kanser", "microsoft/BioGPT-Large-PubMedQA", 25, 5, "tr"],
+    ["Covid-19 est", "microsoft/biogpt", 25, 5, "fr"],
 ]
 
 title = " BioGPT: Generative Pre-trained Transformer for Biomedical Text Generation and Mining"

requirements.txt

@@ -1,3 +1,4 @@
 transformers==4.26.0
 sacremoses
-torch
+torch
+multilingual_translation

utils.py

@@ -0,0 +1,106 @@
+from bs4 import BeautifulSoup
+import requests
+
+
+lang_ids = {
+    "Afrikaans": "af",
+    "Amharic": "am",
+    "Arabic": "ar",
+    "Asturian": "ast",
+    "Azerbaijani": "az",
+    "Bashkir": "ba",
+    "Belarusian": "be",
+    "Bulgarian": "bg",
+    "Bengali": "bn",
+    "Breton": "br",
+    "Bosnian": "bs",
+    "Catalan": "ca",
+    "Cebuano": "ceb",
+    "Czech": "cs",
+    "Welsh": "cy",
+    "Danish": "da",
+    "German": "de",
+    "Greeek": "el",
+    "English": "en",
+    "Spanish": "es",
+    "Estonian": "et",
+    "Persian": "fa",
+    "Fulah": "ff",
+    "Finnish": "fi",
+    "French": "fr",
+    "Western Frisian": "fy",
+    "Irish": "ga",
+    "Gaelic": "gd",
+    "Galician": "gl",
+    "Gujarati": "gu",
+    "Hausa": "ha",
+    "Hebrew": "he",
+    "Hindi": "hi",
+    "Croatian": "hr",
+    "Haitian": "ht",
+    "Hungarian": "hu",
+    "Armenian": "hy",
+    "Indonesian": "id",
+    "Igbo": "ig",
+    "Iloko": "ilo",
+    "Icelandic": "is",
+    "Italian": "it",
+    "Japanese": "ja",
+    "Javanese": "jv",
+    "Georgian": "ka",
+    "Kazakh": "kk",
+    "Central Khmer": "km",
+    "Kannada": "kn",
+    "Korean": "ko",
+    "Luxembourgish": "lb",
+    "Ganda": "lg",
+    "Lingala": "ln",
+    "Lao": "lo",
+    "Lithuanian": "lt",
+    "Latvian": "lv",
+    "Malagasy": "mg",
+    "Macedonian": "mk",
+    "Malayalam": "ml",
+    "Mongolian": "mn",
+    "Marathi": "mr",
+    "Malay": "ms",
+    "Burmese": "my",
+    "Nepali": "ne",
+    "Dutch": "nl",
+    "Norwegian": "no",
+    "Northern Sotho": "ns",
+    "Occitan": "oc",
+    "Oriya": "or",
+    "Panjabi": "pa",
+    "Polish": "pl",
+    "Pushto": "ps",
+    "Portuguese": "pt",
+    "Romanian": "ro",
+    "Russian": "ru",
+    "Sindhi": "sd",
+    "Sinhala": "si",
+    "Slovak": "sk",
+    "Slovenian": "sl",
+    "Somali": "so",
+    "Albanian": "sq",
+    "Serbian": "sr",
+    "Swati": "ss",
+    "Sundanese": "su",
+    "Swedish": "sv",
+    "Swahili": "sw",
+    "Tamil": "ta",
+    "Thai": "th",
+    "Tagalog": "tl",
+    "Tswana": "tn",
+    "Turkish": "tr",
+    "Ukrainian": "uk",
+    "Urdu": "ur",
+    "Uzbek": "uz",
+    "Vietnamese": "vi",
+    "Wolof": "wo",
+    "Xhosa": "xh",
+    "Yiddish": "yi",
+    "Yoruba": "yo",
+    "Chinese": "zh",
+    "Zulu": "zu",
+}