cointegrated
commited on
Commit
•
60894da
1
Parent(s):
a391bcd
Create README.md
Browse files
README.md
ADDED
@@ -0,0 +1,87 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
---
|
2 |
+
language:
|
3 |
+
- myv
|
4 |
+
- ru
|
5 |
+
- fi
|
6 |
+
- de
|
7 |
+
- es
|
8 |
+
- en
|
9 |
+
- hi
|
10 |
+
- zh
|
11 |
+
- tr
|
12 |
+
- uk
|
13 |
+
- fr
|
14 |
+
- ar
|
15 |
+
tags:
|
16 |
+
- erzya
|
17 |
+
- mordovian
|
18 |
+
- translation
|
19 |
+
license: cc-by-sa-4.0
|
20 |
+
datasets:
|
21 |
+
- slone/myv_ru_2022
|
22 |
+
- yhavinga/ccmatrix
|
23 |
+
---
|
24 |
+
|
25 |
+
This a model to translate texts to the Erzya language (`myv`, cyrillic script) from 11 other languages: `ru,fi,de,es,en,hi,zh,tr,uk,fr,ar`.
|
26 |
+
|
27 |
+
It is described in the paper "The first neural machine translation system for the Erzya language".
|
28 |
+
|
29 |
+
This model is based on [facebook/mbart-large-50](https://huggingface.co/facebook/mbart-large-50) ([license here](https://tfhub.dev/google/LaBSE/2)), but with updated vocabulary and checkpoint:
|
30 |
+
- Added an extra language token `myv_XX` and 19K new BPE tokens for the Erzya language;
|
31 |
+
- Fine-tuned to translate from Erzya: first to Russian, then to all 11 languages.
|
32 |
+
|
33 |
+
The following code can be used to run translation using the model
|
34 |
+
|
35 |
+
```Python
|
36 |
+
from transformers import MBartForConditionalGeneration, MBart50Tokenizer
|
37 |
+
|
38 |
+
|
39 |
+
def fix_tokenizer(tokenizer):
|
40 |
+
""" Add a new language token to the tokenizer vocabulary (this should be done each time after its initialization) """
|
41 |
+
old_len = len(tokenizer) - int('myv_XX' in tokenizer.added_tokens_encoder)
|
42 |
+
tokenizer.lang_code_to_id['myv_XX'] = old_len-1
|
43 |
+
tokenizer.id_to_lang_code[old_len-1] = 'myv_XX'
|
44 |
+
tokenizer.fairseq_tokens_to_ids["<mask>"] = len(tokenizer.sp_model) + len(tokenizer.lang_code_to_id) + tokenizer.fairseq_offset
|
45 |
+
|
46 |
+
tokenizer.fairseq_tokens_to_ids.update(tokenizer.lang_code_to_id)
|
47 |
+
tokenizer.fairseq_ids_to_tokens = {v: k for k, v in tokenizer.fairseq_tokens_to_ids.items()}
|
48 |
+
if 'myv_XX' not in tokenizer._additional_special_tokens:
|
49 |
+
tokenizer._additional_special_tokens.append('myv_XX')
|
50 |
+
tokenizer.added_tokens_encoder = {}
|
51 |
+
|
52 |
+
|
53 |
+
def translate(text, model, tokenizer, src='ru_RU', trg='myv_XX', max_length='auto', num_beams=3, repetition_penalty=5.0, train_mode=False, n_out=None, **kwargs):
|
54 |
+
tokenizer.src_lang = src
|
55 |
+
encoded = tokenizer(text, return_tensors="pt", truncation=True, max_length=1024)
|
56 |
+
if max_length == 'auto':
|
57 |
+
max_length = int(32 + 1.5 * encoded.input_ids.shape[1])
|
58 |
+
if train_mode:
|
59 |
+
model.train()
|
60 |
+
else:
|
61 |
+
model.eval()
|
62 |
+
generated_tokens = model.generate(
|
63 |
+
**encoded.to(model.device),
|
64 |
+
forced_bos_token_id=tokenizer.lang_code_to_id[trg],
|
65 |
+
max_length=max_length,
|
66 |
+
num_beams=num_beams,
|
67 |
+
repetition_penalty=repetition_penalty,
|
68 |
+
num_return_sequences=n_out or 1,
|
69 |
+
**kwargs
|
70 |
+
)
|
71 |
+
out = tokenizer.batch_decode(generated_tokens, skip_special_tokens=True)
|
72 |
+
if isinstance(text, str) and n_out is None:
|
73 |
+
return out[0]
|
74 |
+
return out
|
75 |
+
|
76 |
+
|
77 |
+
mname = 'slone/mbart-large-51-myv-mul-v1'
|
78 |
+
model = MBartForConditionalGeneration.from_pretrained(mname)
|
79 |
+
tokenizer = MBart50Tokenizer.from_pretrained(mname)
|
80 |
+
fix_tokenizer(tokenizer)
|
81 |
+
|
82 |
+
|
83 |
+
print(translate('Шумбрат, киска!', model, tokenizer, src='myv_XX', trg='ru_RU'))
|
84 |
+
# Привет, собака! # действительно, "киска" с эрзянского переводится именно так
|
85 |
+
print(translate('Шумбрат, киска!', model, tokenizer, src='myv_XX', trg='en_XX'))
|
86 |
+
# Hi, dog!
|
87 |
+
```
|