CZWin32768 commited on
Commit
324264b
1 Parent(s): b3e456a
Files changed (5) hide show
  1. README.md +44 -0
  2. config.json +22 -0
  3. pytorch_model.bin +3 -0
  4. sentencepiece.bpe.model +3 -0
  5. tokenizer.json +0 -0
README.md ADDED
@@ -0,0 +1,44 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # XLM-Align
2
+
3
+ **Improving Pretrained Cross-Lingual Language Models via Self-Labeled Word Alignment** (ACL-2021, [paper](https://arxiv.org/pdf/2106.06381.pdf), [github](https://github.com/CZWin32768/XLM-Align))
4
+
5
+ XLM-Align is a pretrained cross-lingual language model that supports 94 languages. See details in our [paper](https://arxiv.org/pdf/2106.06381.pdf).
6
+
7
+ ## Example
8
+
9
+ ```
10
+ model = = AutoModel.from_pretrained("CZWin32768/xlm-align")
11
+ ```
12
+
13
+ ## Evaluation Results
14
+
15
+ XTREME cross-lingual understanding tasks:
16
+
17
+ | Model | POS | NER | XQuAD | MLQA | TyDiQA | XNLI | PAWS-X | Avg |
18
+ |:----:|:----:|:----:|:----:|:-----:|:----:|:-----:|:----:|:----:|
19
+ | XLM-R_base | 75.6 | 61.8 | 71.9 / 56.4 | 65.1 / 47.2 | 55.4 / 38.3 | 75.0 | 84.9 | 66.4 |
20
+ | XLM-Align | **76.0** | **63.7** | **74.7 / 59.0** | **68.1 / 49.8** | **62.1 / 44.8** | **76.2** | **86.8** | **68.9** |
21
+
22
+ ## MD5
23
+
24
+ ```
25
+ b9d214025837250ede2f69c9385f812c config.json
26
+ 6005db708eb4bab5b85fa3976b9db85b pytorch_model.bin
27
+ bf25eb5120ad92ef5c7d8596b5dc4046 sentencepiece.bpe.model
28
+ eedbd60a7268b9fc45981b849664f747 tokenizer.json
29
+ ```
30
+
31
+ ## About
32
+
33
+ Contact: chizewen\@outlook.com
34
+
35
+ BibTeX:
36
+
37
+ ```
38
+ @article{xlmalign,
39
+ title={Improving Pretrained Cross-Lingual Language Models via Self-Labeled Word Alignment},
40
+ author={Zewen Chi and Li Dong and Bo Zheng and Shaohan Huang and Xian-Ling Mao and Heyan Huang and Furu Wei},
41
+ journal={arXiv preprint arXiv:2106.06381},
42
+ year={2021}
43
+ }
44
+ ```
config.json ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "XLMRobertaForMaskedLM"
4
+ ],
5
+ "attention_probs_dropout_prob": 0.1,
6
+ "bos_token_id": 0,
7
+ "eos_token_id": 2,
8
+ "hidden_act": "gelu",
9
+ "hidden_dropout_prob": 0.1,
10
+ "hidden_size": 768,
11
+ "initializer_range": 0.02,
12
+ "intermediate_size": 3072,
13
+ "layer_norm_eps": 1e-05,
14
+ "max_position_embeddings": 514,
15
+ "model_type": "xlm-roberta",
16
+ "num_attention_heads": 12,
17
+ "num_hidden_layers": 12,
18
+ "output_past": true,
19
+ "pad_token_id": 1,
20
+ "type_vocab_size": 1,
21
+ "vocab_size": 250002
22
+ }
pytorch_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:33e66265ea1eba45d3773fee7628e7fa089934a2acbd9ed0dc927c4001350dd0
3
+ size 941840167
sentencepiece.bpe.model ADDED
@@ -0,0 +1,3 @@
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cfc8146abe2a0488e9e2a0c56de7952f7c11ab059eca145a0a727afce0db2865
3
+ size 5069051
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff