Spaces:
Runtime error
Runtime error
Vignesh-10215
commited on
Commit
•
338a53a
1
Parent(s):
fd548db
first commit
Browse files- LICENSE +21 -0
- README.md +2 -13
- app.py +219 -0
- delete.py +44 -0
- requirements.txt +4 -0
LICENSE
ADDED
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
MIT License
|
2 |
+
|
3 |
+
Copyright (c) 2022 Vignesh
|
4 |
+
|
5 |
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
6 |
+
of this software and associated documentation files (the "Software"), to deal
|
7 |
+
in the Software without restriction, including without limitation the rights
|
8 |
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9 |
+
copies of the Software, and to permit persons to whom the Software is
|
10 |
+
furnished to do so, subject to the following conditions:
|
11 |
+
|
12 |
+
The above copyright notice and this permission notice shall be included in all
|
13 |
+
copies or substantial portions of the Software.
|
14 |
+
|
15 |
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16 |
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17 |
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18 |
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19 |
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20 |
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
21 |
+
SOFTWARE.
|
README.md
CHANGED
@@ -1,13 +1,2 @@
|
|
1 |
-
|
2 |
-
|
3 |
-
emoji: 🔥
|
4 |
-
colorFrom: pink
|
5 |
-
colorTo: blue
|
6 |
-
sdk: streamlit
|
7 |
-
sdk_version: 1.9.0
|
8 |
-
app_file: app.py
|
9 |
-
pinned: false
|
10 |
-
license: cc
|
11 |
-
---
|
12 |
-
|
13 |
-
Check out the configuration reference at https://huggingface.co/docs/hub/spaces#reference
|
|
|
1 |
+
# alignments
|
2 |
+
finding alignments of source text and translated text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
app.py
ADDED
@@ -0,0 +1,219 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
from select import devpoll
|
2 |
+
import streamlit as st
|
3 |
+
import os
|
4 |
+
import io
|
5 |
+
from transformers import M2M100Tokenizer, M2M100ForConditionalGeneration
|
6 |
+
import time
|
7 |
+
import json
|
8 |
+
from typing import List
|
9 |
+
import torch
|
10 |
+
import random
|
11 |
+
import logging
|
12 |
+
from transformers import BertTokenizer, BertModel, BertConfig
|
13 |
+
|
14 |
+
|
15 |
+
if torch.cuda.is_available():
|
16 |
+
device = torch.device("cuda:0")
|
17 |
+
else:
|
18 |
+
device = torch.device("cpu")
|
19 |
+
logging.warning("GPU not found, using CPU, translation will be very slow.")
|
20 |
+
|
21 |
+
st.cache(suppress_st_warning=True, allow_output_mutation=True)
|
22 |
+
st.set_page_config(page_title="M2M100 Translator")
|
23 |
+
|
24 |
+
lang_id = {
|
25 |
+
"Afrikaans": "af",
|
26 |
+
"Amharic": "am",
|
27 |
+
"Arabic": "ar",
|
28 |
+
"Asturian": "ast",
|
29 |
+
"Azerbaijani": "az",
|
30 |
+
"Bashkir": "ba",
|
31 |
+
"Belarusian": "be",
|
32 |
+
"Bulgarian": "bg",
|
33 |
+
"Bengali": "bn",
|
34 |
+
"Breton": "br",
|
35 |
+
"Bosnian": "bs",
|
36 |
+
"Catalan": "ca",
|
37 |
+
"Cebuano": "ceb",
|
38 |
+
"Czech": "cs",
|
39 |
+
"Welsh": "cy",
|
40 |
+
"Danish": "da",
|
41 |
+
"German": "de",
|
42 |
+
"Greeek": "el",
|
43 |
+
"English": "en",
|
44 |
+
"Spanish": "es",
|
45 |
+
"Estonian": "et",
|
46 |
+
"Persian": "fa",
|
47 |
+
"Fulah": "ff",
|
48 |
+
"Finnish": "fi",
|
49 |
+
"French": "fr",
|
50 |
+
"Western Frisian": "fy",
|
51 |
+
"Irish": "ga",
|
52 |
+
"Gaelic": "gd",
|
53 |
+
"Galician": "gl",
|
54 |
+
"Gujarati": "gu",
|
55 |
+
"Hausa": "ha",
|
56 |
+
"Hebrew": "he",
|
57 |
+
"Hindi": "hi",
|
58 |
+
"Croatian": "hr",
|
59 |
+
"Haitian": "ht",
|
60 |
+
"Hungarian": "hu",
|
61 |
+
"Armenian": "hy",
|
62 |
+
"Indonesian": "id",
|
63 |
+
"Igbo": "ig",
|
64 |
+
"Iloko": "ilo",
|
65 |
+
"Icelandic": "is",
|
66 |
+
"Italian": "it",
|
67 |
+
"Japanese": "ja",
|
68 |
+
"Javanese": "jv",
|
69 |
+
"Georgian": "ka",
|
70 |
+
"Kazakh": "kk",
|
71 |
+
"Central Khmer": "km",
|
72 |
+
"Kannada": "kn",
|
73 |
+
"Korean": "ko",
|
74 |
+
"Luxembourgish": "lb",
|
75 |
+
"Ganda": "lg",
|
76 |
+
"Lingala": "ln",
|
77 |
+
"Lao": "lo",
|
78 |
+
"Lithuanian": "lt",
|
79 |
+
"Latvian": "lv",
|
80 |
+
"Malagasy": "mg",
|
81 |
+
"Macedonian": "mk",
|
82 |
+
"Malayalam": "ml",
|
83 |
+
"Mongolian": "mn",
|
84 |
+
"Marathi": "mr",
|
85 |
+
"Malay": "ms",
|
86 |
+
"Burmese": "my",
|
87 |
+
"Nepali": "ne",
|
88 |
+
"Dutch": "nl",
|
89 |
+
"Norwegian": "no",
|
90 |
+
"Northern Sotho": "ns",
|
91 |
+
"Occitan": "oc",
|
92 |
+
"Oriya": "or",
|
93 |
+
"Panjabi": "pa",
|
94 |
+
"Polish": "pl",
|
95 |
+
"Pushto": "ps",
|
96 |
+
"Portuguese": "pt",
|
97 |
+
"Romanian": "ro",
|
98 |
+
"Russian": "ru",
|
99 |
+
"Sindhi": "sd",
|
100 |
+
"Sinhala": "si",
|
101 |
+
"Slovak": "sk",
|
102 |
+
"Slovenian": "sl",
|
103 |
+
"Somali": "so",
|
104 |
+
"Albanian": "sq",
|
105 |
+
"Serbian": "sr",
|
106 |
+
"Swati": "ss",
|
107 |
+
"Sundanese": "su",
|
108 |
+
"Swedish": "sv",
|
109 |
+
"Swahili": "sw",
|
110 |
+
"Tamil": "ta",
|
111 |
+
"Thai": "th",
|
112 |
+
"Tagalog": "tl",
|
113 |
+
"Tswana": "tn",
|
114 |
+
"Turkish": "tr",
|
115 |
+
"Ukrainian": "uk",
|
116 |
+
"Urdu": "ur",
|
117 |
+
"Uzbek": "uz",
|
118 |
+
"Vietnamese": "vi",
|
119 |
+
"Wolof": "wo",
|
120 |
+
"Xhosa": "xh",
|
121 |
+
"Yiddish": "yi",
|
122 |
+
"Yoruba": "yo",
|
123 |
+
"Chinese": "zh",
|
124 |
+
"Zulu": "zu",
|
125 |
+
}
|
126 |
+
|
127 |
+
|
128 |
+
@st.cache(suppress_st_warning=True, allow_output_mutation=True)
|
129 |
+
def load_model(
|
130 |
+
pretrained_model: str = "facebook/m2m100_1.2B",
|
131 |
+
cache_dir: str = "models/",
|
132 |
+
bert: str = "bert-base-multilingual-cased",
|
133 |
+
):
|
134 |
+
tokenizer = M2M100Tokenizer.from_pretrained(pretrained_model, cache_dir=cache_dir)
|
135 |
+
model = M2M100ForConditionalGeneration.from_pretrained(
|
136 |
+
pretrained_model, cache_dir=cache_dir
|
137 |
+
).to(device)
|
138 |
+
config = BertConfig.from_pretrained(bert, output_hidden_states=True)
|
139 |
+
bert_tokenizer: str = BertTokenizer.from_pretrained(bert, config=config)
|
140 |
+
bert_model: str = BertModel.from_pretrained(bert).to(device)
|
141 |
+
model.eval()
|
142 |
+
bert_model.eval()
|
143 |
+
return tokenizer, model, bert_tokenizer, bert_model
|
144 |
+
|
145 |
+
|
146 |
+
def find_algnments(
|
147 |
+
source_text, translated_text, bert_tokenizer, bert_model, threshold=0.001
|
148 |
+
):
|
149 |
+
source_tokens = bert_tokenizer(source_text, return_tensors="pt")
|
150 |
+
target_tokens = bert_tokenizer(translated_text, return_tensors="pt")
|
151 |
+
bpe_source_map = []
|
152 |
+
for i in source_text.split():
|
153 |
+
bpe_source_map += len(bert_tokenizer.tokenize(i)) * [i]
|
154 |
+
bpe_target_map = []
|
155 |
+
for i in translated_text.split():
|
156 |
+
bpe_target_map += len(bert_tokenizer.tokenize(i)) * [i]
|
157 |
+
source_embedding = bert_model(**source_tokens).hidden_states[8]
|
158 |
+
target_embedding = bert_model(**target_tokens).hidden_states[8]
|
159 |
+
target_embedding = target_embedding.transpose(-1, -2)
|
160 |
+
source_target_mapping = nn.Softmax(dim=-1)(
|
161 |
+
torch.matmul(source_embedding, target_embedding)
|
162 |
+
)
|
163 |
+
target_source_mapping = nn.Softmax(dim=-1)(
|
164 |
+
torch.matmul(target_embedding, source_embedding)
|
165 |
+
)
|
166 |
+
align_matrix = (source_target_mapping > threshold) * (
|
167 |
+
target_source_mapping > threshold
|
168 |
+
)
|
169 |
+
non_zeros = torch.nonzero(align_matrix)
|
170 |
+
align_words = []
|
171 |
+
for i, j, k in non_zeros:
|
172 |
+
if j + 1 < source_tokens_len - 1 and k + 1 < target_tokens_len - 1:
|
173 |
+
align_words.append([bpe_source_map[j + 1], bpe_target_map[k + 1]])
|
174 |
+
return align_words
|
175 |
+
|
176 |
+
|
177 |
+
st.title("M2M100 Translator")
|
178 |
+
st.write(
|
179 |
+
"M2M100 is a multilingual encoder-decoder (seq-to-seq) model trained for Many-to-Many multilingual translation. It was introduced in this paper https://arxiv.org/abs/2010.11125 and first released in https://github.com/pytorch/fairseq/tree/master/examples/m2m_100 repository. The model that can directly translate between the 9,900 directions of 100 languages.\n"
|
180 |
+
)
|
181 |
+
st.write(
|
182 |
+
"The BERT model was proposed in BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding by Jacob Devlin, Ming-Wei Chang, Kenton Lee and Kristina Toutanova. It’s a bidirectional transformer pretrained using a combination of masked language modeling objective and next sentence prediction on a large corpus comprising the Toronto Book Corpus and Wikipedia."
|
183 |
+
)
|
184 |
+
st.write(
|
185 |
+
" This demo uses the facebook/m2m100_1.2B model. For local inference see https://github.com/ikergarcia1996/Easy-Translate"
|
186 |
+
)
|
187 |
+
st.write("This demo uses bert-base-multilingual-cased ")
|
188 |
+
|
189 |
+
user_input: str = st.text_area(
|
190 |
+
"Input text",
|
191 |
+
height=200,
|
192 |
+
max_chars=5120,
|
193 |
+
)
|
194 |
+
|
195 |
+
source_lang = st.selectbox(label="Source language", options=list(lang_id.keys()))
|
196 |
+
target_lang = st.selectbox(label="Target language", options=list(lang_id.keys()))
|
197 |
+
|
198 |
+
if st.button("Run"):
|
199 |
+
time_start = time.time()
|
200 |
+
tokenizer, model, bert_tokenizer, bert_model = load_model()
|
201 |
+
|
202 |
+
src_lang = lang_id[source_lang]
|
203 |
+
trg_lang = lang_id[target_lang]
|
204 |
+
tokenizer.src_lang = src_lang
|
205 |
+
with torch.no_grad():
|
206 |
+
encoded_input = tokenizer(user_input, return_tensors="pt").to(device)
|
207 |
+
generated_tokens = model.generate(
|
208 |
+
**encoded_input, forced_bos_token_id=tokenizer.get_lang_id(trg_lang)
|
209 |
+
)
|
210 |
+
translated_text = tokenizer.batch_decode(
|
211 |
+
generated_tokens, skip_special_tokens=True
|
212 |
+
)[0]
|
213 |
+
|
214 |
+
time_end = time.time()
|
215 |
+
alignments = find_algnments(user_input, translated_text, bert_tokenizer, bert_model)
|
216 |
+
for i, j in alignments:
|
217 |
+
st.success(f"{i}->{j}")
|
218 |
+
|
219 |
+
st.write(f"Computation time: {round((time_end-time_start),3)} sec")
|
delete.py
ADDED
@@ -0,0 +1,44 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
from transformers import BertTokenizer, BertModel, BertConfig
|
2 |
+
import torch
|
3 |
+
from torch import nn
|
4 |
+
|
5 |
+
threshold = 0.001
|
6 |
+
device = "cpu"
|
7 |
+
bert = "bert-base-multilingual-cased"
|
8 |
+
config = BertConfig.from_pretrained(bert, output_hidden_states=True)
|
9 |
+
bert_tokenizer = BertTokenizer.from_pretrained(bert)
|
10 |
+
bert_model = BertModel.from_pretrained(bert, config=config).to(device)
|
11 |
+
source_text = "Hello, my dog is cute"
|
12 |
+
translated_text = "Hello, my dog is cute"
|
13 |
+
source_tokens = bert_tokenizer(source_text, return_tensors="pt")
|
14 |
+
print(source_tokens)
|
15 |
+
source_tokens_len = len(bert_tokenizer.tokenize(source_text))
|
16 |
+
target_tokens_len = len(bert_tokenizer.tokenize(translated_text))
|
17 |
+
target_tokens = bert_tokenizer(translated_text, return_tensors="pt")
|
18 |
+
bpe_source_map = []
|
19 |
+
for i in source_text.split():
|
20 |
+
bpe_source_map += len(bert_tokenizer.tokenize(i)) * [i]
|
21 |
+
bpe_target_map = []
|
22 |
+
for i in translated_text.split():
|
23 |
+
bpe_target_map += len(bert_tokenizer.tokenize(i)) * [i]
|
24 |
+
source_embedding = bert_model(**source_tokens).hidden_states[8]
|
25 |
+
target_embedding = bert_model(**target_tokens).hidden_states[8]
|
26 |
+
target_embedding = target_embedding.transpose(-1, -2)
|
27 |
+
source_target_mapping = nn.Softmax(dim=-1)(
|
28 |
+
torch.matmul(source_embedding, target_embedding)
|
29 |
+
)
|
30 |
+
print(source_target_mapping.shape)
|
31 |
+
target_source_mapping = nn.Softmax(dim=-2)(
|
32 |
+
torch.matmul(source_embedding, target_embedding)
|
33 |
+
)
|
34 |
+
print(target_source_mapping.shape)
|
35 |
+
|
36 |
+
align_matrix = (source_target_mapping > threshold) * (target_source_mapping > threshold)
|
37 |
+
align_prob = (2 * source_target_mapping * target_source_mapping) / (
|
38 |
+
source_target_mapping + target_source_mapping + 1e-9
|
39 |
+
)
|
40 |
+
non_zeros = torch.nonzero(align_matrix)
|
41 |
+
print(non_zeros)
|
42 |
+
for i, j, k in non_zeros:
|
43 |
+
if j + 1 < source_tokens_len - 1 and k + 1 < target_tokens_len - 1:
|
44 |
+
print(bpe_source_map[j + 1], bpe_target_map[k + 1])
|
requirements.txt
ADDED
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
1 |
+
streamlit
|
2 |
+
torch
|
3 |
+
transformers
|
4 |
+
transformers[sentencepiece]
|