Duplicate from maxbittker/english-tokipona-round-trip-translator

Co-authored-by: max bittker <maxbittker@users.noreply.huggingface.co>

.gitattributes

@@ -0,0 +1,34 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,2 @@
+.venv
+__pycache__

README.md

@@ -0,0 +1,13 @@
+---
+title: English / toki pona Translator
+emoji: 💬
+colorFrom: indigo
+colorTo: purple
+sdk: gradio
+sdk_version: 3.15.0
+app_file: app.py
+pinned: false
+duplicated_from: maxbittker/english-tokipona-round-trip-translator
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,122 @@
+import gradio as gr
+from transformers import AutoTokenizer, AutoModelForSeq2SeqLM, pipeline
+import torch
+
+model = AutoModelForSeq2SeqLM.from_pretrained("Jayyydyyy/m2m100_418m_tokipona")
+tokenizer = AutoTokenizer.from_pretrained("facebook/m2m100_418M")
+device = "cuda:0" if torch.cuda.is_available() else "cpu"
+model.to(device)
+LANG_CODES = {"English": "en", "toki pona": "tl"}
+
+
+def translate(text):
+    """
+    Translate the text from source lang to target lang
+    """
+
+    # src = LANG_CODES.get(src_lang)
+    # tgt = LANG_CODES.get(tgt_lang)
+
+    tokenizer.src_lang = "en"
+    tokenizer.tgt_lang = "tl"
+    ins = tokenizer(text, return_tensors="pt").to(device)
+
+    gen_args = {
+        "return_dict_in_generate": True,
+        "output_scores": True,
+        "output_hidden_states": True,
+        "length_penalty": 0.0,  # don't encourage longer or shorter output,
+        "num_return_sequences": 1,
+        "num_beams": 1,
+        "forced_bos_token_id": tokenizer.lang_code_to_id["tl"],
+    }
+
+    outs = model.generate(**{**ins, **gen_args})
+    output = tokenizer.batch_decode(outs.sequences, skip_special_tokens=True)
+    text2 = "\n".join(output)
+
+    ##################
+
+    tokenizer.src_lang = "tl"
+    tokenizer.tgt_lang = "en"
+
+    ins = tokenizer(text2, return_tensors="pt").to(device)
+
+    gen_args = {
+        "return_dict_in_generate": True,
+        "output_scores": True,
+        "output_hidden_states": True,
+        "length_penalty": 0.0,  # don't encourage longer or shorter output,
+        "num_return_sequences": 1,
+        "num_beams": 1,
+        "forced_bos_token_id": tokenizer.lang_code_to_id["en"],
+    }
+
+    outs2 = model.generate(**{**ins, **gen_args})
+    output2 = tokenizer.batch_decode(outs2.sequences, skip_special_tokens=True)
+
+    return "\n".join(output2)
+
+
+with gr.Blocks() as app:
+    markdown = """
+    # An English / toki pona Neural Machine Translation App!
+    
+    ### toki a! 💬
+
+    This is an english to toki pona / toki pona to english neural machine translation app.
+
+    Input your text to translate, a source language and target language, and desired number of return sequences! 
+
+    ### Grammar Regularization
+    An interesting quirk of training a many-to-many translation model is that pseudo-grammar correction 
+    can be achieved by translating *from* **language A** *to* **language A**
+    
+    Remember, this can ***approximate*** grammaticality, but it isn't always the best.
+    
+    For example, "mi li toki e toki pona" (Source Language: toki pona & Target Language: toki pona) will result in: 
+    - ['mi toki e toki pona.', 'mi toki pona.', 'mi toki e toki pona']
+    - (Thus, the ungrammatical "li" is dropped)
+
+    ### Model and Data
+    This app utilizes a fine-tuned version of Facebook/Meta AI's M2M100 418M param model. 
+    
+    By leveraging the pretrained weights of the massively multilingual M2M100 model, 
+    we can jumpstart our transfer learning to accomplish machine translation for toki pona!
+    
+    The model was fine-tuned on the English/toki pona bitexts found at [https://tatoeba.org/](https://tatoeba.org/)
+    
+    ### This app is a work in progress and obviously not all translations will be perfect. 
+    In addition to parameter quantity and the hyper-parameters used while training, 
+    the *quality of data* found on Tatoeba directly influences the perfomance of projects like this! 
+
+    If you wish to contribute, please add high quality and diverse translations to Tatoeba!
+    """
+
+    with gr.Row():
+        gr.Markdown(markdown)
+        with gr.Column():
+            input_text = gr.components.Textbox(
+                label="Input Text",
+                value="Raccoons are fascinating creatures, but I prefer opossums.",
+            )
+            # source_lang = gr.components.Dropdown(label="Source Language", value="English", choices=list(LANG_CODES.keys()))
+            # target_lang = gr.components.Dropdown(label="Target Language", value="toki pona", choices=list(LANG_CODES.keys()))
+            # return_seqs = gr.Slider(label="Number of return sequences", value=3, minimum=1, maximum=12, step=1)
+
+            inputs = [input_text]
+            outputs = gr.Textbox()
+
+            translate_btn = gr.Button("Translate! | o ante toki!")
+            translate_btn.click(translate, inputs=inputs, outputs=outputs, api_name="translate")
+
+            gr.Examples(
+                [
+                    ["Hello! How are you?", "English", "toki pona", 3],
+                    ["toki a! ilo pi ante toki ni li pona!", "toki pona", "English", 3],
+                    ["mi li toki e toki pona", "toki pona", "toki pona", 3],
+                ],
+                inputs=inputs,
+            )
+
+app.launch()

requirements.txt

@@ -0,0 +1,4 @@
+torch
+transformers
+gradio
+sentencepiece