Initial commit

.gitignore

@@ -0,0 +1,5 @@
+venv
+models
+**/__pycache__/ 
+hf_cache
+.env

README.md

@@ -0,0 +1,11 @@
+---
+title: Translator
+emoji: 🌎
+colorFrom: red
+colorTo: pink
+sdk: gradio
+sdk_version: 4.8.0
+app_file: app.py
+pinned: false
+license: apache-2.0
+---

app.py

@@ -0,0 +1,220 @@
+import gc
+import os
+from dotenv import load_dotenv
+import gradio as gr
+from AinaTheme import AinaGradioTheme
+import sentencepiece as spm
+import ctranslate2
+from huggingface_hub import snapshot_download
+import nltk
+from nltk import sent_tokenize
+
+nltk.download('punkt')
+
+load_dotenv()
+
+MODELS_PATH = "./models"
+HF_CACHE_DIR = "./hf_cache"
+MAX_INPUT_CHARACTERS= int(os.environ.get("MAX_INPUT_CHARACTERS", default=500))
+
+def download_model(repo_id, revision="main"):
+    return snapshot_download(repo_id=repo_id, revision=revision, local_dir=os.path.join(MODELS_PATH, repo_id), cache_dir=HF_CACHE_DIR)
+
+model_dir_ca_es = download_model("projecte-aina/mt-aina-ca-es", revision="main")
+model_dir_es_ca = download_model("PlanTL-GOB-ES/mt-plantl-es-ca", revision="main")
+
+model_dir_ca_en = download_model("projecte-aina/mt-aina-ca-en", revision="main")
+model_dir_en_ca = download_model("projecte-aina/mt-aina-en-ca", revision="main")
+
+model_dir_ca_fr = download_model("projecte-aina/mt-aina-ca-fr", revision="main")
+model_dir_fr_ca = download_model("projecte-aina/mt-aina-fr-ca", revision="main")
+
+model_dir_ca_de = download_model("projecte-aina/mt-aina-ca-de", revision="main")
+model_dir_de_ca = download_model("projecte-aina/mt-aina-de-ca", revision="main")
+
+model_dir_ca_it = download_model("projecte-aina/mt-aina-ca-it", revision="main")
+model_dir_it_ca = download_model("projecte-aina/mt-aina-it-ca", revision="main")
+
+model_dir_ca_pt = download_model("projecte-aina/mt-aina-ca-pt", revision="main")
+model_dir_pt_ca = download_model("projecte-aina/mt-aina-pt-ca", revision="main")
+
+directions = {
+    "Catalan": {
+        "target": {
+            "Spanish": {"model": (f"{os.path.join(MODELS_PATH, model_dir_ca_es)}/spm.model", f"{os.path.join(MODELS_PATH, model_dir_ca_es)}")},
+            "English": {"model": (f"{os.path.join(MODELS_PATH, model_dir_ca_en)}/spm.model", f"{os.path.join(MODELS_PATH, model_dir_ca_en)}")},
+            "French": {"model": (f"{os.path.join(MODELS_PATH, model_dir_ca_fr)}/spm.model", f"{os.path.join(MODELS_PATH, model_dir_ca_fr)}")},
+            "German": {"model": (f"{os.path.join(MODELS_PATH, model_dir_ca_de)}/spm.model", f"{os.path.join(MODELS_PATH, model_dir_ca_de)}")},
+            "Italian": {"model": (f"{os.path.join(MODELS_PATH, model_dir_ca_it)}/spm.model", f"{os.path.join(MODELS_PATH, model_dir_ca_it)}")},
+            "Portuguese": {"model": (f"{os.path.join(MODELS_PATH, model_dir_ca_pt)}/spm.model", f"{os.path.join(MODELS_PATH, model_dir_ca_pt)}")}
+
+        }
+    },
+    "Spanish": {
+        "target": {
+            "Catalan": {"model": (f"{os.path.join(MODELS_PATH, model_dir_es_ca)}/spm.model", f"{os.path.join(MODELS_PATH, model_dir_es_ca)}")},
+        }
+    },
+    "English": {
+        "target": {
+            "Catalan": {"model": (f"{os.path.join(MODELS_PATH, model_dir_en_ca)}/spm.model", f"{os.path.join(MODELS_PATH, model_dir_en_ca)}")},
+        }
+    },
+    "French": {
+        "target": {
+            "Catalan": {"model": (f"{os.path.join(MODELS_PATH, model_dir_fr_ca)}/spm.model", f"{os.path.join(MODELS_PATH, model_dir_fr_ca)}")},
+        }
+    },
+    "German": {
+        "target": {
+            "Catalan": {"model": (f"{os.path.join(MODELS_PATH, model_dir_de_ca)}/spm.model", f"{os.path.join(MODELS_PATH, model_dir_de_ca)}")},
+        }
+    },
+    "Italian": {
+        "target": {
+            "Catalan": {"model": (f"{os.path.join(MODELS_PATH, model_dir_it_ca)}/spm.model", f"{os.path.join(MODELS_PATH, model_dir_it_ca)}")},
+        }
+    },
+    "Portuguese": {
+        "target": {
+            "Catalan": {"model": (f"{os.path.join(MODELS_PATH, model_dir_pt_ca)}/spm.model", f"{os.path.join(MODELS_PATH, model_dir_pt_ca)}")},
+        }
+    }
+}
+
+DEFAULT_SOURCE_LANGUAGE = list(directions.keys())[0]
+
+def get_target_languages(source_language):
+    return list(directions.get(source_language, {}).get("target", {}).keys())
+
+
+def get_target_languege_model(source_language, target_language):
+    return directions.get(source_language, {}).get("target", {}).get(target_language, {}).get("model")
+
+
+def translate(source, lang_pair):
+    """Use CTranslate model to translate a sentence
+
+    Args:
+        source (str): Source sentences to translate
+        translator (object): Object of Translator, with the CTranslate2 model
+        sp_model (object): Object of SentencePieceProcessor, with the SentencePiece source model
+    Returns:
+        Translation of the source text
+    """
+    sp_model = spm.SentencePieceProcessor(lang_pair[0])
+    translator = ctranslate2.Translator(lang_pair[1])
+
+    source_sentences = sent_tokenize(source)
+    source_tokenized = sp_model.encode(source_sentences, out_type=str)
+    translations = translator.translate_batch(source_tokenized)
+    translations = [translation[0]["tokens"] for translation in translations]
+    translations_detokenized = sp_model.decode(translations)
+    translation = " ".join(translations_detokenized)
+    translation = translation.replace(' ⁇', ':')
+
+    gc.collect()
+
+    return translation
+
+
+def translate_input(input, source_language, target_language):
+
+    if input.strip() == "":  
+        gr.Warning('Not possible to translate an empty input.')
+        return None
+    
+    target_language_model = get_target_languege_model(source_language, target_language)
+    translation = translate(input, target_language_model)
+
+    return translation
+
+
+def clear():
+    return None, None
+
+def change_interactive(text):
+    if len(text.strip()) > MAX_INPUT_CHARACTERS:
+        return gr.update(interactive = True),  gr.update(interactive = False)
+    return gr.update(interactive = True),  gr.update(interactive = True)    
+
+def update_target_languages_dropdown(source_language):
+
+    output_languages = get_target_languages(source_language)
+    return gr.update(choices=output_languages, value=output_languages[0], interactive=True)
+
+
+with gr.Blocks(**AinaGradioTheme().get_kwargs()) as app:
+    with gr.Row(variant="panel"):    
+        with gr.Column(scale=2):
+            placeholder_max_token = gr.Textbox(
+                    visible=False,
+                    interactive=False,
+                    value= MAX_INPUT_CHARACTERS
+                )
+            source_language = gr.Dropdown(label="Source Language", choices=list(directions.keys()), value=DEFAULT_SOURCE_LANGUAGE)
+            input = gr.Textbox(placeholder="Enter a text here to translate.",  max_lines=100, lines=12, show_label=False, interactive=True)
+            with gr.Row(variant="panel", equal_height=True):
+                    gr.HTML("""<span id="countertext" style="display: flex; justify-content: start; color:#ef4444; font-weight: bold;"></span>""")
+                    gr.HTML(f"""<span id="counter" style="display: flex; justify-content: end;"> <span id="inputlenght">0</span>&nbsp;/&nbsp;{MAX_INPUT_CHARACTERS}</span>""")
+
+        with gr.Column(scale=2):
+
+            target_outputs = get_target_languages(DEFAULT_SOURCE_LANGUAGE)
+            #target_language = gr.Dropdown(choices=target_outputs, label="Target Language", value=target_outputs[0])
+
+            target_language = gr.Radio(choices=target_outputs, label="Target Language", value=target_outputs[0])
+            output = gr.Textbox(max_lines=100, lines=12, show_label=False, interactive=False, show_copy_button=True)
+
+            with gr.Row(variant="panel"):
+                    clear_btn = gr.Button(
+                        "Clear", 
+                    )
+                    submit_btn = gr.Button(
+                        "Submit", 
+                        variant="primary",
+                    )
+
+        source_language.change(fn=update_target_languages_dropdown, inputs=[source_language], outputs=target_language)
+
+        input.change(
+            fn=change_interactive, 
+            inputs=[input], 
+            outputs=[clear_btn, submit_btn],
+            api_name=False
+        )
+      
+        input.change(
+            fn=None, 
+            inputs=[input], 
+            js=f"""(i) => document.getElementById('countertext').textContent =  i.length > {MAX_INPUT_CHARACTERS} && 'Max length {MAX_INPUT_CHARACTERS} characters. ' || '' """,
+            api_name=False
+        )
+
+        input.change(
+            fn=None, 
+            inputs=[input, placeholder_max_token], 
+            js="""(i, m) => {
+                document.getElementById('inputlenght').textContent = i.length + '  '
+                document.getElementById('inputlenght').style.color =  (i.length > m) ? "#ef4444" : "";
+            }""",
+            api_name=False
+            )
+    
+        clear_btn.click(
+            fn=clear, 
+            inputs=[], 
+            outputs=[input, output], 
+            queue=False,
+            api_name=False
+        )
+        
+        submit_btn.click(
+            fn=translate_input, 
+            inputs=[input, source_language, target_language], 
+            outputs=[output],
+            api_name="translate",
+            concurrency_limit=1,
+        )
+
+app.launch(show_api=True)

requirements.txt

@@ -0,0 +1,6 @@
+aina-gradio-theme==1.0
+gradio==4.8.0
+ctranslate2==3.23.0
+nltk==3.8.1
+sentencepiece==0.1.99
+python-dotenv==1.0.0