Commit
•
c9d55c0
1
Parent(s):
4b35fe0
Upload 10 files
Browse files- README.md +60 -3
- added_tokens.json +1 -0
- config.json +21 -0
- flax_model.msgpack +3 -0
- gitattributes +17 -0
- pytorch_model.bin +3 -0
- special_tokens_map.json +1 -0
- spiece.model +3 -0
- tf_model.h5 +3 -0
- tokenizer_config.json +1 -0
README.md
CHANGED
@@ -1,3 +1,60 @@
|
|
1 |
-
---
|
2 |
-
|
3 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
---
|
2 |
+
language: "mn"
|
3 |
+
tags:
|
4 |
+
- bert
|
5 |
+
- mongolian
|
6 |
+
- uncased
|
7 |
+
---
|
8 |
+
|
9 |
+
# BERT-BASE-MONGOLIAN-UNCASED
|
10 |
+
[Link to Official Mongolian-BERT repo](https://github.com/tugstugi/mongolian-bert)
|
11 |
+
|
12 |
+
## Model description
|
13 |
+
This repository contains pre-trained Mongolian [BERT](https://arxiv.org/abs/1810.04805) models trained by [tugstugi](https://github.com/tugstugi), [enod](https://github.com/enod) and [sharavsambuu](https://github.com/sharavsambuu).
|
14 |
+
Special thanks to [nabar](https://github.com/nabar) who provided 5x TPUs.
|
15 |
+
|
16 |
+
This repository is based on the following open source projects: [google-research/bert](https://github.com/google-research/bert/),
|
17 |
+
[huggingface/pytorch-pretrained-BERT](https://github.com/huggingface/pytorch-pretrained-BERT) and [yoheikikuta/bert-japanese](https://github.com/yoheikikuta/bert-japanese).
|
18 |
+
|
19 |
+
#### How to use
|
20 |
+
|
21 |
+
```python
|
22 |
+
from transformers import pipeline, AutoTokenizer, AutoModelForMaskedLM
|
23 |
+
|
24 |
+
tokenizer = AutoTokenizer.from_pretrained('tugstugi/bert-base-mongolian-uncased', use_fast=False)
|
25 |
+
model = AutoModelForMaskedLM.from_pretrained('tugstugi/bert-base-mongolian-uncased')
|
26 |
+
|
27 |
+
## declare task ##
|
28 |
+
pipe = pipeline(task="fill-mask", model=model, tokenizer=tokenizer)
|
29 |
+
|
30 |
+
## example ##
|
31 |
+
input_ = 'Миний [MASK] хоол идэх нь тун чухал.'
|
32 |
+
|
33 |
+
output_ = pipe(input_)
|
34 |
+
for i in range(len(output_)):
|
35 |
+
print(output_[i])
|
36 |
+
|
37 |
+
## output ##
|
38 |
+
#{'sequence': 'миний хувьд хоол идэх нь тун чухал.', 'score': 0.7889143824577332, 'token': 126, 'token_str': 'хувьд'}
|
39 |
+
#{'sequence': 'миний бодлоор хоол идэх нь тун чухал.', 'score': 0.18616807460784912, 'token': 6106, 'token_str': 'бодлоор'}
|
40 |
+
#{'sequence': 'миний зүгээс хоол идэх нь тун чухал.', 'score': 0.004825591575354338, 'token': 761, 'token_str': 'зүгээс'}
|
41 |
+
#{'sequence': 'миний биед хоол идэх нь тун чухал.', 'score': 0.0015743684489279985, 'token': 3010, 'token_str': 'биед'}
|
42 |
+
#{'sequence': 'миний тухайд хоол идэх нь тун чухал.', 'score': 0.0014919431414455175, 'token': 1712, 'token_str': 'тухайд'}
|
43 |
+
```
|
44 |
+
|
45 |
+
|
46 |
+
## Training data
|
47 |
+
Mongolian Wikipedia and the 700 million word Mongolian news data set [[Pretraining Procedure](https://github.com/tugstugi/mongolian-bert#pre-training)]
|
48 |
+
|
49 |
+
### BibTeX entry and citation info
|
50 |
+
|
51 |
+
```bibtex
|
52 |
+
@misc{mongolian-bert,
|
53 |
+
author = {Tuguldur, Erdene-Ochir and Gunchinish, Sharavsambuu and Bataa, Enkhbold},
|
54 |
+
title = {BERT Pretrained Models on Mongolian Datasets},
|
55 |
+
year = {2019},
|
56 |
+
publisher = {GitHub},
|
57 |
+
journal = {GitHub repository},
|
58 |
+
howpublished = {\url{https://github.com/tugstugi/mongolian-bert/}}
|
59 |
+
}
|
60 |
+
```
|
added_tokens.json
ADDED
@@ -0,0 +1 @@
|
|
|
|
|
1 |
+
{"<pad>": 27000}
|
config.json
ADDED
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"architectures": [
|
3 |
+
"BertForMaskedLM"
|
4 |
+
],
|
5 |
+
"attention_probs_dropout_prob": 0.1,
|
6 |
+
"gradient_checkpointing": false,
|
7 |
+
"hidden_act": "gelu",
|
8 |
+
"hidden_dropout_prob": 0.1,
|
9 |
+
"hidden_size": 768,
|
10 |
+
"initializer_range": 0.02,
|
11 |
+
"intermediate_size": 3072,
|
12 |
+
"layer_norm_eps": 1e-12,
|
13 |
+
"max_position_embeddings": 512,
|
14 |
+
"model_type": "bert",
|
15 |
+
"num_attention_heads": 12,
|
16 |
+
"num_hidden_layers": 12,
|
17 |
+
"pad_token_id": 0,
|
18 |
+
"type_vocab_size": 2,
|
19 |
+
"vocab_size": 27000,
|
20 |
+
"tokenizer_class": "AlbertTokenizer"
|
21 |
+
}
|
flax_model.msgpack
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:32ff915d5f935669180fa9ab000a8ecb97df80441e983af48b82022d3b008531
|
3 |
+
size 442610787
|
gitattributes
ADDED
@@ -0,0 +1,17 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
*.bin.* filter=lfs diff=lfs merge=lfs -text
|
2 |
+
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
3 |
+
*.bin filter=lfs diff=lfs merge=lfs -text
|
4 |
+
*.h5 filter=lfs diff=lfs merge=lfs -text
|
5 |
+
*.tflite filter=lfs diff=lfs merge=lfs -text
|
6 |
+
*.tar.gz filter=lfs diff=lfs merge=lfs -text
|
7 |
+
*.ot filter=lfs diff=lfs merge=lfs -text
|
8 |
+
*.onnx filter=lfs diff=lfs merge=lfs -text
|
9 |
+
*.arrow filter=lfs diff=lfs merge=lfs -text
|
10 |
+
*.ftz filter=lfs diff=lfs merge=lfs -text
|
11 |
+
*.joblib filter=lfs diff=lfs merge=lfs -text
|
12 |
+
*.model filter=lfs diff=lfs merge=lfs -text
|
13 |
+
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
14 |
+
*.pb filter=lfs diff=lfs merge=lfs -text
|
15 |
+
*.pt filter=lfs diff=lfs merge=lfs -text
|
16 |
+
*.pth filter=lfs diff=lfs merge=lfs -text
|
17 |
+
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
pytorch_model.bin
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:866380e7756622c93c94b7ffc28cf036b3e621f5e2f596b2d7886c55577d771b
|
3 |
+
size 445054385
|
special_tokens_map.json
ADDED
@@ -0,0 +1 @@
|
|
|
|
|
1 |
+
{"bos_token": "[CLS]", "eos_token": "[SEP]", "unk_token": "<unk>", "sep_token": "[SEP]", "pad_token": "<pad>", "cls_token": "[CLS]", "mask_token": "[MASK]"}
|
spiece.model
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:19914d955b6d028592b0a331a43b7235b860e5e4964e46e8c004a21eee0b2b27
|
3 |
+
size 1007226
|
tf_model.h5
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:d07d2e1fdb92e85536c0330f5f5c7b46dd0304b3e6722868bd6b2df47a65c7e4
|
3 |
+
size 545136208
|
tokenizer_config.json
ADDED
@@ -0,0 +1 @@
|
|
|
|
|
1 |
+
{"keep_accents": true, "do_lower_case": true}
|