sultan commited on
Commit
32a05a3
1 Parent(s): 3b1d6d8

Uploading the Model

Browse files
Files changed (5) hide show
  1. PubMD-30k-clean.vocab +0 -0
  2. README.md +40 -0
  3. config.json +30 -0
  4. pytorch_model.bin +3 -0
  5. spiece.model +3 -0
PubMD-30k-clean.vocab ADDED
The diff for this file is too large to render. See raw diff
 
README.md ADDED
@@ -0,0 +1,40 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ BioM-Transformers: Building Large Biomedical Language Models with
2
+ BERT, ALBERT and ELECTRA
3
+
4
+ Abstract
5
+
6
+
7
+ The impact of design choices on the performance
8
+ of biomedical language models recently
9
+ has been a subject for investigation. In
10
+ this paper, we empirically study biomedical
11
+ domain adaptation with large transformer models
12
+ using different design choices. We evaluate
13
+ the performance of our pretrained models
14
+ against other existing biomedical language
15
+ models in the literature. Our results show that
16
+ we achieve state-of-the-art results on several
17
+ biomedical domain tasks despite using similar
18
+ or less computational cost compared to other
19
+ models in the literature. Our findings highlight
20
+ the significant effect of design choices on
21
+ improving the performance of biomedical language
22
+ models.
23
+
24
+ This model was pre-trained on PMC full article for further 64k steps with a batch size of 8192, where we initiate our weights from our model BioM-ALBERT-xxlarge.
25
+
26
+ ```bibtex
27
+ @inproceedings{alrowili-shanker-2021-biom,
28
+ title = "{B}io{M}-Transformers: Building Large Biomedical Language Models with {BERT}, {ALBERT} and {ELECTRA}",
29
+ author = "Alrowili, Sultan and
30
+ Shanker, Vijay",
31
+ booktitle = "Proceedings of the 20th Workshop on Biomedical Language Processing",
32
+ month = jun,
33
+ year = "2021",
34
+ address = "Online",
35
+ publisher = "Association for Computational Linguistics",
36
+ url = "https://www.aclweb.org/anthology/2021.bionlp-1.24",
37
+ pages = "221--227",
38
+ abstract = "The impact of design choices on the performance of biomedical language models recently has been a subject for investigation. In this paper, we empirically study biomedical domain adaptation with large transformer models using different design choices. We evaluate the performance of our pretrained models against other existing biomedical language models in the literature. Our results show that we achieve state-of-the-art results on several biomedical domain tasks despite using similar or less computational cost compared to other models in the literature. Our findings highlight the significant effect of design choices on improving the performance of biomedical language models.",
39
+ }
40
+ ```
config.json ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "AlbertForMaskedLM"
4
+ ],
5
+ "attention_probs_dropout_prob": 0,
6
+ "bos_token_id": 2,
7
+ "classifier_dropout_prob": 0.1,
8
+ "down_scale_factor": 1,
9
+ "embedding_size": 128,
10
+ "eos_token_id": 3,
11
+ "gap_size": 0,
12
+ "hidden_act": "gelu",
13
+ "hidden_dropout_prob": 0,
14
+ "hidden_size": 4096,
15
+ "initializer_range": 0.02,
16
+ "inner_group_num": 1,
17
+ "intermediate_size": 16384,
18
+ "layer_norm_eps": 1e-12,
19
+ "layers_to_keep": [],
20
+ "max_position_embeddings": 512,
21
+ "model_type": "albert",
22
+ "net_structure_type": 0,
23
+ "num_attention_heads": 64,
24
+ "num_hidden_groups": 1,
25
+ "num_hidden_layers": 12,
26
+ "num_memory_blocks": 0,
27
+ "pad_token_id": 0,
28
+ "type_vocab_size": 2,
29
+ "vocab_size": 30000
30
+ }
pytorch_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b1a8721125b5bd77d895df67acc5ece1609bffc8226706e16350c34d1e0cfc9d
3
+ size 892651736
spiece.model ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a677d8e80a34a6ac26b1d54e436a83d073e2991f27e7110df1039f751830ce95
3
+ size 778401