yapzanan commited on
Commit
0ee3b61
1 Parent(s): 1233deb

Upload 4 files

Browse files
Files changed (4) hide show
  1. app.py +51 -0
  2. flores200_codes.py +215 -0
  3. requirements.txt +4 -0
  4. test.py +70 -0
app.py ADDED
@@ -0,0 +1,51 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from flask import Flask, request, jsonify
2
+ import torch
3
+ from transformers import AutoTokenizer, AutoModelForSeq2SeqLM, pipeline
4
+ from flores200_codes import flores_codes
5
+ from flask_cors import CORS
6
+ app = Flask(__name__)
7
+
8
+ model_dict = {}
9
+ CORS(app)
10
+
11
+ TASK = "translation"
12
+ CKPT = "facebook/nllb-200-distilled-600M"
13
+
14
+ model = AutoModelForSeq2SeqLM.from_pretrained(CKPT)
15
+ tokenizer = AutoTokenizer.from_pretrained(CKPT)
16
+
17
+ device = 0 if torch.cuda.is_available() else -1
18
+
19
+
20
+ def translate_text(text, src_lang, tgt_lang, max_length=400):
21
+ """
22
+ Translate the text from source lang to target lang
23
+ """
24
+ translation_pipeline = pipeline(TASK,
25
+ model=model,
26
+ tokenizer=tokenizer,
27
+ src_lang=src_lang,
28
+ tgt_lang=tgt_lang,
29
+ max_length=max_length,
30
+ device=device)
31
+
32
+ result = translation_pipeline(text)
33
+ return result[0]['translation_text']
34
+
35
+
36
+ @app.route("/translate", methods=["POST"])
37
+ def handle_translate():
38
+ data = request.get_json()
39
+ source = data.get("source")
40
+ target = data.get("target")
41
+ text = data.get("text")
42
+ print(source, target, text)
43
+ result = translate_text(text, source, target)
44
+ return jsonify(result)
45
+
46
+ @app.route("/languages", methods=["GET"])
47
+ def getlanguages():
48
+ return jsonify(list(flores_codes.keys()))
49
+
50
+ if __name__ == "__main__":
51
+ app.run(debug=True)
flores200_codes.py ADDED
@@ -0,0 +1,215 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ codes_as_string = '''Acehnese (Arabic script) ace_Arab
2
+ Acehnese (Latin script) ace_Latn
3
+ Mesopotamian Arabic acm_Arab
4
+ Ta’izzi-Adeni Arabic acq_Arab
5
+ Tunisian Arabic aeb_Arab
6
+ Afrikaans afr_Latn
7
+ South Levantine Arabic ajp_Arab
8
+ Akan aka_Latn
9
+ Amharic amh_Ethi
10
+ North Levantine Arabic apc_Arab
11
+ Modern Standard Arabic arb_Arab
12
+ Modern Standard Arabic (Romanized) arb_Latn
13
+ Najdi Arabic ars_Arab
14
+ Moroccan Arabic ary_Arab
15
+ Egyptian Arabic arz_Arab
16
+ Assamese asm_Beng
17
+ Asturian ast_Latn
18
+ Awadhi awa_Deva
19
+ Central Aymara ayr_Latn
20
+ South Azerbaijani azb_Arab
21
+ North Azerbaijani azj_Latn
22
+ Bashkir bak_Cyrl
23
+ Bambara bam_Latn
24
+ Balinese ban_Latn
25
+ Belarusian bel_Cyrl
26
+ Bemba bem_Latn
27
+ Bengali ben_Beng
28
+ Bhojpuri bho_Deva
29
+ Banjar (Arabic script) bjn_Arab
30
+ Banjar (Latin script) bjn_Latn
31
+ Standard Tibetan bod_Tibt
32
+ Bosnian bos_Latn
33
+ Buginese bug_Latn
34
+ Bulgarian bul_Cyrl
35
+ Catalan cat_Latn
36
+ Cebuano ceb_Latn
37
+ Czech ces_Latn
38
+ Chokwe cjk_Latn
39
+ Central Kurdish ckb_Arab
40
+ Crimean Tatar crh_Latn
41
+ Welsh cym_Latn
42
+ Danish dan_Latn
43
+ German deu_Latn
44
+ Southwestern Dinka dik_Latn
45
+ Dyula dyu_Latn
46
+ Dzongkha dzo_Tibt
47
+ Greek ell_Grek
48
+ English eng_Latn
49
+ Esperanto epo_Latn
50
+ Estonian est_Latn
51
+ Basque eus_Latn
52
+ Ewe ewe_Latn
53
+ Faroese fao_Latn
54
+ Fijian fij_Latn
55
+ Finnish fin_Latn
56
+ Fon fon_Latn
57
+ French fra_Latn
58
+ Friulian fur_Latn
59
+ Nigerian Fulfulde fuv_Latn
60
+ Scottish Gaelic gla_Latn
61
+ Irish gle_Latn
62
+ Galician glg_Latn
63
+ Guarani grn_Latn
64
+ Gujarati guj_Gujr
65
+ Haitian Creole hat_Latn
66
+ Hausa hau_Latn
67
+ Hebrew heb_Hebr
68
+ Hindi hin_Deva
69
+ Chhattisgarhi hne_Deva
70
+ Croatian hrv_Latn
71
+ Hungarian hun_Latn
72
+ Armenian hye_Armn
73
+ Igbo ibo_Latn
74
+ Ilocano ilo_Latn
75
+ Indonesian ind_Latn
76
+ Icelandic isl_Latn
77
+ Italian ita_Latn
78
+ Javanese jav_Latn
79
+ Japanese jpn_Jpan
80
+ Kabyle kab_Latn
81
+ Jingpho kac_Latn
82
+ Kamba kam_Latn
83
+ Kannada kan_Knda
84
+ Kashmiri (Arabic script) kas_Arab
85
+ Kashmiri (Devanagari script) kas_Deva
86
+ Georgian kat_Geor
87
+ Central Kanuri (Arabic script) knc_Arab
88
+ Central Kanuri (Latin script) knc_Latn
89
+ Kazakh kaz_Cyrl
90
+ Kabiyè kbp_Latn
91
+ Kabuverdianu kea_Latn
92
+ Khmer khm_Khmr
93
+ Kikuyu kik_Latn
94
+ Kinyarwanda kin_Latn
95
+ Kyrgyz kir_Cyrl
96
+ Kimbundu kmb_Latn
97
+ Northern Kurdish kmr_Latn
98
+ Kikongo kon_Latn
99
+ Korean kor_Hang
100
+ Lao lao_Laoo
101
+ Ligurian lij_Latn
102
+ Limburgish lim_Latn
103
+ Lingala lin_Latn
104
+ Lithuanian lit_Latn
105
+ Lombard lmo_Latn
106
+ Latgalian ltg_Latn
107
+ Luxembourgish ltz_Latn
108
+ Luba-Kasai lua_Latn
109
+ Ganda lug_Latn
110
+ Luo luo_Latn
111
+ Mizo lus_Latn
112
+ Standard Latvian lvs_Latn
113
+ Magahi mag_Deva
114
+ Maithili mai_Deva
115
+ Malayalam mal_Mlym
116
+ Marathi mar_Deva
117
+ Minangkabau (Arabic script) min_Arab
118
+ Minangkabau (Latin script) min_Latn
119
+ Macedonian mkd_Cyrl
120
+ Plateau Malagasy plt_Latn
121
+ Maltese mlt_Latn
122
+ Meitei (Bengali script) mni_Beng
123
+ Halh Mongolian khk_Cyrl
124
+ Mossi mos_Latn
125
+ Maori mri_Latn
126
+ Burmese mya_Mymr
127
+ Dutch nld_Latn
128
+ Norwegian Nynorsk nno_Latn
129
+ Norwegian Bokmål nob_Latn
130
+ Nepali npi_Deva
131
+ Northern Sotho nso_Latn
132
+ Nuer nus_Latn
133
+ Nyanja nya_Latn
134
+ Occitan oci_Latn
135
+ West Central Oromo gaz_Latn
136
+ Odia ory_Orya
137
+ Pangasinan pag_Latn
138
+ Eastern Panjabi pan_Guru
139
+ Papiamento pap_Latn
140
+ Western Persian pes_Arab
141
+ Polish pol_Latn
142
+ Portuguese por_Latn
143
+ Dari prs_Arab
144
+ Southern Pashto pbt_Arab
145
+ Ayacucho Quechua quy_Latn
146
+ Romanian ron_Latn
147
+ Rundi run_Latn
148
+ Russian rus_Cyrl
149
+ Sango sag_Latn
150
+ Sanskrit san_Deva
151
+ Santali sat_Olck
152
+ Sicilian scn_Latn
153
+ Shan shn_Mymr
154
+ Sinhala sin_Sinh
155
+ Slovak slk_Latn
156
+ Slovenian slv_Latn
157
+ Samoan smo_Latn
158
+ Shona sna_Latn
159
+ Sindhi snd_Arab
160
+ Somali som_Latn
161
+ Southern Sotho sot_Latn
162
+ Spanish spa_Latn
163
+ Tosk Albanian als_Latn
164
+ Sardinian srd_Latn
165
+ Serbian srp_Cyrl
166
+ Swati ssw_Latn
167
+ Sundanese sun_Latn
168
+ Swedish swe_Latn
169
+ Swahili swh_Latn
170
+ Silesian szl_Latn
171
+ Tamil tam_Taml
172
+ Tatar tat_Cyrl
173
+ Telugu tel_Telu
174
+ Tajik tgk_Cyrl
175
+ Tagalog tgl_Latn
176
+ Thai tha_Thai
177
+ Tigrinya tir_Ethi
178
+ Tamasheq (Latin script) taq_Latn
179
+ Tamasheq (Tifinagh script) taq_Tfng
180
+ Tok Pisin tpi_Latn
181
+ Tswana tsn_Latn
182
+ Tsonga tso_Latn
183
+ Turkmen tuk_Latn
184
+ Tumbuka tum_Latn
185
+ Turkish tur_Latn
186
+ Twi twi_Latn
187
+ Central Atlas Tamazight tzm_Tfng
188
+ Uyghur uig_Arab
189
+ Ukrainian ukr_Cyrl
190
+ Umbundu umb_Latn
191
+ Urdu urd_Arab
192
+ Northern Uzbek uzn_Latn
193
+ Venetian vec_Latn
194
+ Vietnamese vie_Latn
195
+ Waray war_Latn
196
+ Wolof wol_Latn
197
+ Xhosa xho_Latn
198
+ Eastern Yiddish ydd_Hebr
199
+ Yoruba yor_Latn
200
+ Yue Chinese yue_Hant
201
+ Chinese (Simplified) zho_Hans
202
+ Chinese (Traditional) zho_Hant
203
+ Standard Malay zsm_Latn
204
+ Zulu zul_Latn'''
205
+
206
+ codes_as_string = codes_as_string.split('\n')
207
+
208
+ flores_codes = {}
209
+ for code in codes_as_string:
210
+ lang, lang_code = code.split('\t')
211
+ flores_codes[lang_code] = lang_code
212
+
213
+
214
+
215
+
requirements.txt ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ git+https://github.com/huggingface/transformers
2
+ gradio
3
+ torch
4
+ flask
test.py ADDED
@@ -0,0 +1,70 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from flask import Flask, request, jsonify
2
+ import torch
3
+ from transformers import AutoTokenizer, AutoModelForSeq2SeqLM, pipeline
4
+ from flores200_codes import flores_codes
5
+ from flask_cors import CORS
6
+ app = Flask(__name__)
7
+
8
+ model_dict = {}
9
+ CORS(app, origins=["http://localhost:3000"])
10
+
11
+
12
+ def load_models():
13
+ # build model and tokenizer
14
+ model_name_dict = {
15
+ 'nllb-distilled-600M': 'facebook/nllb-200-distilled-600M',
16
+ # 'nllb-1.3B': 'facebook/nllb-200-1.3B',
17
+ # 'nllb-distilled-1.3B': 'facebook/nllb-200-distilled-1.3B',
18
+ # 'nllb-3.3B': 'facebook/nllb-200-3.3B',
19
+ }
20
+
21
+ for call_name, real_name in model_name_dict.items():
22
+ print('\tLoading model: %s' % call_name)
23
+ model = AutoModelForSeq2SeqLM.from_pretrained(real_name)
24
+ tokenizer = AutoTokenizer.from_pretrained(real_name)
25
+ model_dict[call_name+'_model'] = model
26
+ model_dict[call_name+'_tokenizer'] = tokenizer
27
+
28
+
29
+ def translation(source, target, text):
30
+ source = flores_codes[source]
31
+ target = flores_codes[target]
32
+
33
+ model_name = 'nllb-distilled-600M'
34
+ model = model_dict[model_name + '_model']
35
+ tokenizer = model_dict[model_name + '_tokenizer']
36
+
37
+ translator = pipeline('translation', model=model, tokenizer=tokenizer, src_lang=source, tgt_lang=target)
38
+ output = translator(text, max_length=400)
39
+
40
+ full_output = output
41
+ output = output[0]['translation_text']
42
+ result = {
43
+ 'source': source,
44
+ 'target': target,
45
+ 'result': output,
46
+ 'full_output': full_output
47
+ }
48
+ return result
49
+
50
+
51
+ @app.route("/translate", methods=["POST"])
52
+ def translate():
53
+ data = request.get_json()
54
+ source = data.get("source")
55
+ target = data.get("target")
56
+ text = data.get("text")
57
+ print(source, target, text)
58
+ result = translation(source, target, text)
59
+ return jsonify(result)
60
+
61
+
62
+ @app.route("/languages", methods=["GET"])
63
+ def getlanguages():
64
+ return jsonify(list(flores_codes.keys()))
65
+
66
+
67
+ if __name__ == '__main__':
68
+ print('\tinit models')
69
+ load_models()
70
+ app.run()