maxbittker Jayyydyyy commited on
Commit
d510d4c
0 Parent(s):

Duplicate from Jayyydyyy/english-tokipona-translator

Browse files

Co-authored-by: Jordan Myers <Jayyydyyy@users.noreply.huggingface.co>

Files changed (5) hide show
  1. .gitattributes +34 -0
  2. .gitignore +2 -0
  3. README.md +13 -0
  4. app.py +100 -0
  5. requirements.txt +4 -0
.gitattributes ADDED
@@ -0,0 +1,34 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ *.7z filter=lfs diff=lfs merge=lfs -text
2
+ *.arrow filter=lfs diff=lfs merge=lfs -text
3
+ *.bin filter=lfs diff=lfs merge=lfs -text
4
+ *.bz2 filter=lfs diff=lfs merge=lfs -text
5
+ *.ckpt filter=lfs diff=lfs merge=lfs -text
6
+ *.ftz filter=lfs diff=lfs merge=lfs -text
7
+ *.gz filter=lfs diff=lfs merge=lfs -text
8
+ *.h5 filter=lfs diff=lfs merge=lfs -text
9
+ *.joblib filter=lfs diff=lfs merge=lfs -text
10
+ *.lfs.* filter=lfs diff=lfs merge=lfs -text
11
+ *.mlmodel filter=lfs diff=lfs merge=lfs -text
12
+ *.model filter=lfs diff=lfs merge=lfs -text
13
+ *.msgpack filter=lfs diff=lfs merge=lfs -text
14
+ *.npy filter=lfs diff=lfs merge=lfs -text
15
+ *.npz filter=lfs diff=lfs merge=lfs -text
16
+ *.onnx filter=lfs diff=lfs merge=lfs -text
17
+ *.ot filter=lfs diff=lfs merge=lfs -text
18
+ *.parquet filter=lfs diff=lfs merge=lfs -text
19
+ *.pb filter=lfs diff=lfs merge=lfs -text
20
+ *.pickle filter=lfs diff=lfs merge=lfs -text
21
+ *.pkl filter=lfs diff=lfs merge=lfs -text
22
+ *.pt filter=lfs diff=lfs merge=lfs -text
23
+ *.pth filter=lfs diff=lfs merge=lfs -text
24
+ *.rar filter=lfs diff=lfs merge=lfs -text
25
+ *.safetensors filter=lfs diff=lfs merge=lfs -text
26
+ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
27
+ *.tar.* filter=lfs diff=lfs merge=lfs -text
28
+ *.tflite filter=lfs diff=lfs merge=lfs -text
29
+ *.tgz filter=lfs diff=lfs merge=lfs -text
30
+ *.wasm filter=lfs diff=lfs merge=lfs -text
31
+ *.xz filter=lfs diff=lfs merge=lfs -text
32
+ *.zip filter=lfs diff=lfs merge=lfs -text
33
+ *.zst filter=lfs diff=lfs merge=lfs -text
34
+ *tfevents* filter=lfs diff=lfs merge=lfs -text
.gitignore ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ .venv
2
+ __pycache__
README.md ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ title: English / toki pona Translator
3
+ emoji: 💬
4
+ colorFrom: indigo
5
+ colorTo: purple
6
+ sdk: gradio
7
+ sdk_version: 3.15.0
8
+ app_file: app.py
9
+ pinned: false
10
+ duplicated_from: Jayyydyyy/english-tokipona-translator
11
+ ---
12
+
13
+ Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
app.py ADDED
@@ -0,0 +1,100 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import gradio as gr
2
+ from transformers import AutoTokenizer, AutoModelForSeq2SeqLM, pipeline
3
+ import torch
4
+
5
+ model = AutoModelForSeq2SeqLM.from_pretrained("Jayyydyyy/m2m100_418m_tokipona")
6
+ tokenizer = AutoTokenizer.from_pretrained("facebook/m2m100_418M")
7
+ device = "cuda:0" if torch.cuda.is_available() else "cpu"
8
+ LANG_CODES = {
9
+ "English":"en",
10
+ "toki pona":"tl"
11
+ }
12
+
13
+ def translate(text, src_lang, tgt_lang, candidates:int):
14
+ """
15
+ Translate the text from source lang to target lang
16
+ """
17
+
18
+ src = LANG_CODES.get(src_lang)
19
+ tgt = LANG_CODES.get(tgt_lang)
20
+
21
+ tokenizer.src_lang = src
22
+ tokenizer.tgt_lang = tgt
23
+
24
+ ins = tokenizer(text, return_tensors='pt').to(device)
25
+
26
+ gen_args = {
27
+ 'return_dict_in_generate': True,
28
+ 'output_scores': True,
29
+ 'output_hidden_states': True,
30
+ 'length_penalty': 0.0, # don't encourage longer or shorter output,
31
+ 'num_return_sequences': candidates,
32
+ 'num_beams':candidates,
33
+ 'forced_bos_token_id': tokenizer.lang_code_to_id[tgt]
34
+ }
35
+
36
+
37
+ outs = model.generate(**{**ins, **gen_args})
38
+ output = tokenizer.batch_decode(outs.sequences, skip_special_tokens=True)
39
+
40
+ return '\n'.join(output)
41
+
42
+ with gr.Blocks() as app:
43
+ markdown="""
44
+ # An English / toki pona Neural Machine Translation App!
45
+
46
+ ### toki a! 💬
47
+
48
+ This is an english to toki pona / toki pona to english neural machine translation app.
49
+
50
+ Input your text to translate, a source language and target language, and desired number of return sequences!
51
+
52
+ ### Grammar Regularization
53
+ An interesting quirk of training a many-to-many translation model is that pseudo-grammar correction
54
+ can be achieved by translating *from* **language A** *to* **language A**
55
+
56
+ Remember, this can ***approximate*** grammaticality, but it isn't always the best.
57
+
58
+ For example, "mi li toki e toki pona" (Source Language: toki pona & Target Language: toki pona) will result in:
59
+ - ['mi toki e toki pona.', 'mi toki pona.', 'mi toki e toki pona']
60
+ - (Thus, the ungrammatical "li" is dropped)
61
+
62
+ ### Model and Data
63
+ This app utilizes a fine-tuned version of Facebook/Meta AI's M2M100 418M param model.
64
+
65
+ By leveraging the pretrained weights of the massively multilingual M2M100 model,
66
+ we can jumpstart our transfer learning to accomplish machine translation for toki pona!
67
+
68
+ The model was fine-tuned on the English/toki pona bitexts found at [https://tatoeba.org/](https://tatoeba.org/)
69
+
70
+ ### This app is a work in progress and obviously not all translations will be perfect.
71
+ In addition to parameter quantity and the hyper-parameters used while training,
72
+ the *quality of data* found on Tatoeba directly influences the perfomance of projects like this!
73
+
74
+ If you wish to contribute, please add high quality and diverse translations to Tatoeba!
75
+ """
76
+
77
+ with gr.Row():
78
+ gr.Markdown(markdown)
79
+ with gr.Column():
80
+ input_text = gr.components.Textbox(label="Input Text", value="Raccoons are fascinating creatures, but I prefer opossums.")
81
+ source_lang = gr.components.Dropdown(label="Source Language", value="English", choices=list(LANG_CODES.keys()))
82
+ target_lang = gr.components.Dropdown(label="Target Language", value="toki pona", choices=list(LANG_CODES.keys()))
83
+ return_seqs = gr.Slider(label="Number of return sequences", value=3, minimum=1, maximum=12, step=1)
84
+
85
+ inputs=[input_text, source_lang, target_lang, return_seqs]
86
+ outputs = gr.Textbox()
87
+
88
+ translate_btn = gr.Button("Translate! | o ante toki!")
89
+ translate_btn.click(translate, inputs=inputs, outputs=outputs)
90
+
91
+ gr.Examples(
92
+ [
93
+ ["Hello! How are you?", "English", "toki pona", 3],
94
+ ["toki a! ilo pi ante toki ni li pona!", "toki pona", "English", 3],
95
+ ["mi li toki e toki pona", "toki pona", "toki pona", 3],
96
+ ],
97
+ inputs=inputs
98
+ )
99
+
100
+ app.launch()
requirements.txt ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ torch
2
+ transformers
3
+ gradio
4
+ sentencepiece