kamalkraj commited on
Commit
0848dac
1 Parent(s): 2949384

base model

Browse files
Files changed (7) hide show
  1. README.md +38 -0
  2. bpe_encoder.bin +3 -0
  3. config.json +30 -0
  4. merges.txt +0 -0
  5. tf_model.h5 +3 -0
  6. tokenizer_config.json +4 -0
  7. vocab.json +0 -0
README.md ADDED
@@ -0,0 +1,38 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ language: en
3
+ tags: deberta-v1
4
+ thumbnail: https://huggingface.co/front/thumbnails/microsoft.png
5
+ license: mit
6
+ ---
7
+
8
+ ## DeBERTa: Decoding-enhanced BERT with Disentangled Attention
9
+
10
+ [DeBERTa](https://arxiv.org/abs/2006.03654) improves the BERT and RoBERTa models using disentangled attention and enhanced mask decoder. It outperforms BERT and RoBERTa on majority of NLU tasks with 80GB training data.
11
+
12
+ Please check the [official repository](https://github.com/microsoft/DeBERTa) for more details and updates.
13
+
14
+
15
+ #### Fine-tuning on NLU tasks
16
+
17
+ We present the dev results on SQuAD 1.1/2.0 and MNLI tasks.
18
+
19
+ | Model | SQuAD 1.1 | SQuAD 2.0 | MNLI-m |
20
+ |-------------------|-----------|-----------|--------|
21
+ | RoBERTa-base | 91.5/84.6 | 83.7/80.5 | 87.6 |
22
+ | XLNet-Large | -/- | -/80.2 | 86.8 |
23
+ | **DeBERTa-base** | 93.1/87.2 | 86.2/83.1 | 88.8 |
24
+
25
+ ### Citation
26
+
27
+ If you find DeBERTa useful for your work, please cite the following paper:
28
+
29
+ ``` latex
30
+ @inproceedings{
31
+ he2021deberta,
32
+ title={DEBERTA: DECODING-ENHANCED BERT WITH DISENTANGLED ATTENTION},
33
+ author={Pengcheng He and Xiaodong Liu and Jianfeng Gao and Weizhu Chen},
34
+ booktitle={International Conference on Learning Representations},
35
+ year={2021},
36
+ url={https://openreview.net/forum?id=XPZIaotutsD}
37
+ }
38
+ ```
bpe_encoder.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e7c6f9eecb461c01e09c00656ccf3e27944b9e74bfe29e51632b13d3cd9d6c8e
3
+ size 3917897
config.json ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "DebertaModel"
4
+ ],
5
+ "attention_probs_dropout_prob": 0.1,
6
+ "hidden_act": "gelu",
7
+ "hidden_dropout_prob": 0.1,
8
+ "hidden_size": 768,
9
+ "initializer_range": 0.02,
10
+ "intermediate_size": 3072,
11
+ "layer_norm_eps": 1e-07,
12
+ "max_position_embeddings": 512,
13
+ "max_relative_positions": -1,
14
+ "model_type": "deberta",
15
+ "num_attention_heads": 12,
16
+ "num_hidden_layers": 12,
17
+ "pad_token_id": 0,
18
+ "pooler_dropout": 0,
19
+ "pooler_hidden_act": "gelu",
20
+ "pooler_hidden_size": 768,
21
+ "pos_att_type": [
22
+ "c2p",
23
+ "p2c"
24
+ ],
25
+ "position_biased_input": false,
26
+ "relative_attention": true,
27
+ "transformers_version": "4.10.0.dev0",
28
+ "type_vocab_size": 0,
29
+ "vocab_size": 50265
30
+ }
merges.txt ADDED
The diff for this file is too large to render. See raw diff
 
tf_model.h5 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:68a9b33a7a94717d8d3ccde814f479185ce9257f8230ec5c2ce1aeadfc96407b
3
+ size 554669920
tokenizer_config.json ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ {
2
+ "do_lower_case": false,
3
+ "vocab_type": "gpt2"
4
+ }
vocab.json ADDED
The diff for this file is too large to render. See raw diff