davda54
commited on
Commit
•
b98ad83
1
Parent(s):
1896fcb
upload
Browse files- config.json +27 -0
- configuration_ltgbert.py +107 -0
- modeling_ltgbert.py +797 -0
- pytorch_model.bin +3 -0
- special_tokens_map.json +9 -0
- tokenizer.json +0 -0
- tokenizer_config.json +4 -0
config.json
ADDED
@@ -0,0 +1,27 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"architectures": [
|
3 |
+
"LtgBertForMaskedLM"
|
4 |
+
],
|
5 |
+
"attention_probs_dropout_prob": 0.1,
|
6 |
+
"auto_map": {
|
7 |
+
"AutoConfig": "configuration_ltgbert.LtgBertConfig",
|
8 |
+
"AutoModel": "modeling_ltgbert.LtgBertModel",
|
9 |
+
"AutoModelForMaskedLM": "modeling_ltgbert.LtgBertForMaskedLM",
|
10 |
+
"AutoModelForSequenceClassification": "modeling_ltgbert.LtgBertForSequenceClassification"
|
11 |
+
},
|
12 |
+
"classifier_dropout": 0.2,
|
13 |
+
"hidden_dropout_prob": 0.1,
|
14 |
+
"hidden_size": 768,
|
15 |
+
"intermediate_size": 2048,
|
16 |
+
"layer_norm_eps": 1e-07,
|
17 |
+
"max_position_embeddings": 512,
|
18 |
+
"model_type": "ltgbert",
|
19 |
+
"num_attention_heads": 12,
|
20 |
+
"num_hidden_layers": 12,
|
21 |
+
"output_all_encoded_layers": true,
|
22 |
+
"pad_token_id": 4,
|
23 |
+
"position_bucket_size": 32,
|
24 |
+
"torch_dtype": "float32",
|
25 |
+
"transformers_version": "4.26.0",
|
26 |
+
"vocab_size": 16384
|
27 |
+
}
|
configuration_ltgbert.py
ADDED
@@ -0,0 +1,107 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
# coding=utf-8
|
2 |
+
# Copyright 2023 Language Technology Group from University of Oslo and The HuggingFace Inc. team.
|
3 |
+
#
|
4 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5 |
+
# you may not use this file except in compliance with the License.
|
6 |
+
# You may obtain a copy of the License at
|
7 |
+
#
|
8 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9 |
+
#
|
10 |
+
# Unless required by applicable law or agreed to in writing, software
|
11 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13 |
+
# See the License for the specific language governing permissions and
|
14 |
+
# limitations under the License.
|
15 |
+
|
16 |
+
""" LTG-BERT configutation """
|
17 |
+
|
18 |
+
|
19 |
+
from transformers.configuration_utils import PretrainedConfig
|
20 |
+
|
21 |
+
|
22 |
+
LTG_BERT_PRETRAINED_CONFIG_ARCHIVE_MAP = {
|
23 |
+
"bnc-bert-span": "https://huggingface.co/ltg/bnc-bert-span",
|
24 |
+
"bnc-bert-span-2x": "https://huggingface.co/ltg/bnc-bert-span-2x",
|
25 |
+
"bnc-bert-span-0.5x": "https://huggingface.co/ltg/bnc-bert-span-0.5x",
|
26 |
+
"bnc-bert-span-0.25x": "https://huggingface.co/ltg/bnc-bert-span-0.25x",
|
27 |
+
"bnc-bert-span-order": "https://huggingface.co/ltg/bnc-bert-span-order",
|
28 |
+
"bnc-bert-span-document": "https://huggingface.co/ltg/bnc-bert-span-document",
|
29 |
+
"bnc-bert-span-word": "https://huggingface.co/ltg/bnc-bert-span-word",
|
30 |
+
"bnc-bert-span-subword": "https://huggingface.co/ltg/bnc-bert-span-subword",
|
31 |
+
|
32 |
+
"norbert3-xs": "https://huggingface.co/ltg/norbert3-xs/config.json",
|
33 |
+
"norbert3-small": "https://huggingface.co/ltg/norbert3-small/config.json",
|
34 |
+
"norbert3-base": "https://huggingface.co/ltg/norbert3-base/config.json",
|
35 |
+
"norbert3-large": "https://huggingface.co/ltg/norbert3-large/config.json",
|
36 |
+
|
37 |
+
"norbert3-oversampled-base": "https://huggingface.co/ltg/norbert3-oversampled-base/config.json",
|
38 |
+
"norbert3-ncc-base": "https://huggingface.co/ltg/norbert3-ncc-base/config.json",
|
39 |
+
"norbert3-nak-base": "https://huggingface.co/ltg/norbert3-nak-base/config.json",
|
40 |
+
"norbert3-nb-base": "https://huggingface.co/ltg/norbert3-nb-base/config.json",
|
41 |
+
"norbert3-wiki-base": "https://huggingface.co/ltg/norbert3-wiki-base/config.json",
|
42 |
+
"norbert3-c4-base": "https://huggingface.co/ltg/norbert3-c4-base/config.json"
|
43 |
+
}
|
44 |
+
|
45 |
+
|
46 |
+
class LtgBertConfig(PretrainedConfig):
|
47 |
+
r"""
|
48 |
+
This is the configuration class to store the configuration of a [`LtgBertModel`]. It is used to
|
49 |
+
instantiate an LTG-BERT model according to the specified arguments, defining the model architecture.
|
50 |
+
Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
|
51 |
+
documentation from [`PretrainedConfig`] for more information.
|
52 |
+
Args:
|
53 |
+
vocab_size (`int`, *optional*, defaults to 16384):
|
54 |
+
Vocabulary size of the LTG-BERT model. Defines the number of different tokens that can be represented by the
|
55 |
+
`inputs_ids` passed when calling [`LtgBertModel`].
|
56 |
+
hidden_size (`int`, *optional*, defaults to 768):
|
57 |
+
Dimensionality of the encoder layers and the pooler layer.
|
58 |
+
num_hidden_layers (`int`, *optional*, defaults to 12):
|
59 |
+
Number of hidden layers in the Transformer encoder.
|
60 |
+
num_attention_heads (`int`, *optional*, defaults to 12):
|
61 |
+
Number of attention heads for each attention layer in the Transformer encoder.
|
62 |
+
intermediate_size (`int`, *optional*, defaults to 2048):
|
63 |
+
Dimensionality of the "intermediate" (often named feed-forward) layer in the Transformer encoder.
|
64 |
+
hidden_dropout_prob (`float`, *optional*, defaults to 0.1):
|
65 |
+
The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.
|
66 |
+
attention_probs_dropout_prob (`float`, *optional*, defaults to 0.1):
|
67 |
+
The dropout ratio for the attention probabilities.
|
68 |
+
max_position_embeddings (`int`, *optional*, defaults to 512):
|
69 |
+
The maximum sequence length that this model might ever be used with. Typically set this to something large
|
70 |
+
just in case (e.g., 512 or 1024 or 2048).
|
71 |
+
layer_norm_eps (`float`, *optional*, defaults to 1e-12):
|
72 |
+
The epsilon used by the layer normalization layers.
|
73 |
+
classifier_dropout (`float`, *optional*):
|
74 |
+
The dropout ratio for the classification head.
|
75 |
+
"""
|
76 |
+
model_type = "ltgbert"
|
77 |
+
def __init__(
|
78 |
+
self,
|
79 |
+
vocab_size=16384,
|
80 |
+
attention_probs_dropout_prob=0.1,
|
81 |
+
hidden_dropout_prob=0.1,
|
82 |
+
hidden_size=768,
|
83 |
+
intermediate_size=2048,
|
84 |
+
max_position_embeddings=512,
|
85 |
+
position_bucket_size=32,
|
86 |
+
num_attention_heads=12,
|
87 |
+
num_hidden_layers=12,
|
88 |
+
layer_norm_eps=1.0e-7,
|
89 |
+
pad_token_id=4,
|
90 |
+
output_all_encoded_layers=True,
|
91 |
+
classifier_dropout=None,
|
92 |
+
**kwargs,
|
93 |
+
):
|
94 |
+
super().__init__(pad_token_id=pad_token_id, **kwargs)
|
95 |
+
|
96 |
+
self.vocab_size = vocab_size
|
97 |
+
self.hidden_size = hidden_size
|
98 |
+
self.num_hidden_layers = num_hidden_layers
|
99 |
+
self.num_attention_heads = num_attention_heads
|
100 |
+
self.intermediate_size = intermediate_size
|
101 |
+
self.hidden_dropout_prob = hidden_dropout_prob
|
102 |
+
self.attention_probs_dropout_prob = attention_probs_dropout_prob
|
103 |
+
self.max_position_embeddings = max_position_embeddings
|
104 |
+
self.output_all_encoded_layers = output_all_encoded_layers
|
105 |
+
self.position_bucket_size = position_bucket_size
|
106 |
+
self.layer_norm_eps = layer_norm_eps
|
107 |
+
self.classifier_dropout = classifier_dropout
|
modeling_ltgbert.py
ADDED
@@ -0,0 +1,797 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
# coding=utf-8
|
2 |
+
# Copyright 2023 Language Technology Group from University of Oslo and The HuggingFace Inc. team.
|
3 |
+
#
|
4 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5 |
+
# you may not use this file except in compliance with the License.
|
6 |
+
# You may obtain a copy of the License at
|
7 |
+
#
|
8 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9 |
+
#
|
10 |
+
# Unless required by applicable law or agreed to in writing, software
|
11 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13 |
+
# See the License for the specific language governing permissions and
|
14 |
+
# limitations under the License.
|
15 |
+
|
16 |
+
""" PyTorch LTG-BERT model."""
|
17 |
+
|
18 |
+
|
19 |
+
import math
|
20 |
+
from typing import List, Optional, Tuple, Union
|
21 |
+
|
22 |
+
import torch
|
23 |
+
import torch.nn as nn
|
24 |
+
import torch.nn.functional as F
|
25 |
+
from torch.utils import checkpoint
|
26 |
+
|
27 |
+
from .configuration_ltgbert import LtgBertConfig
|
28 |
+
from transformers.modeling_utils import PreTrainedModel
|
29 |
+
from transformers.activations import gelu_new
|
30 |
+
from transformers.modeling_outputs import (
|
31 |
+
MaskedLMOutput,
|
32 |
+
MultipleChoiceModelOutput,
|
33 |
+
QuestionAnsweringModelOutput,
|
34 |
+
SequenceClassifierOutput,
|
35 |
+
TokenClassifierOutput,
|
36 |
+
BaseModelOutput
|
37 |
+
)
|
38 |
+
from transformers.pytorch_utils import softmax_backward_data
|
39 |
+
from transformers.utils import add_start_docstrings, add_start_docstrings_to_model_forward
|
40 |
+
|
41 |
+
|
42 |
+
_CHECKPOINT_FOR_DOC = "ltg/bnc-bert-span"
|
43 |
+
_CONFIG_FOR_DOC = "LtgBertConfig"
|
44 |
+
|
45 |
+
|
46 |
+
LTG_BERT_PRETRAINED_MODEL_ARCHIVE_LIST = [
|
47 |
+
"bnc-bert-span",
|
48 |
+
"bnc-bert-span-2x",
|
49 |
+
"bnc-bert-span-0.5x",
|
50 |
+
"bnc-bert-span-0.25x",
|
51 |
+
"bnc-bert-span-order",
|
52 |
+
"bnc-bert-span-document",
|
53 |
+
"bnc-bert-span-word",
|
54 |
+
"bnc-bert-span-subword",
|
55 |
+
|
56 |
+
"norbert3-xs",
|
57 |
+
"norbert3-small",
|
58 |
+
"norbert3-base",
|
59 |
+
"norbert3-large",
|
60 |
+
|
61 |
+
"norbert3-oversampled-base",
|
62 |
+
"norbert3-ncc-base",
|
63 |
+
"norbert3-nak-base",
|
64 |
+
"norbert3-nb-base",
|
65 |
+
"norbert3-wiki-base",
|
66 |
+
"norbert3-c4-base"
|
67 |
+
]
|
68 |
+
|
69 |
+
|
70 |
+
class Encoder(nn.Module):
|
71 |
+
def __init__(self, config, activation_checkpointing=False):
|
72 |
+
super().__init__()
|
73 |
+
self.layers = nn.ModuleList([EncoderLayer(config) for _ in range(config.num_hidden_layers)])
|
74 |
+
|
75 |
+
for i, layer in enumerate(self.layers):
|
76 |
+
layer.mlp.mlp[1].weight.data *= math.sqrt(1.0 / (2.0 * (1 + i)))
|
77 |
+
layer.mlp.mlp[-2].weight.data *= math.sqrt(1.0 / (2.0 * (1 + i)))
|
78 |
+
|
79 |
+
self.activation_checkpointing = activation_checkpointing
|
80 |
+
|
81 |
+
def forward(self, hidden_states, attention_mask, relative_embedding):
|
82 |
+
hidden_states, attention_probs = [hidden_states], []
|
83 |
+
|
84 |
+
for layer in self.layers:
|
85 |
+
if self.activation_checkpointing:
|
86 |
+
hidden_state, attention_p = checkpoint.checkpoint(layer, hidden_states[-1], attention_mask, relative_embedding)
|
87 |
+
else:
|
88 |
+
hidden_state, attention_p = layer(hidden_states[-1], attention_mask, relative_embedding)
|
89 |
+
|
90 |
+
hidden_states.append(hidden_state)
|
91 |
+
attention_probs.append(attention_p)
|
92 |
+
|
93 |
+
return hidden_states, attention_probs
|
94 |
+
|
95 |
+
|
96 |
+
class MaskClassifier(nn.Module):
|
97 |
+
def __init__(self, config, subword_embedding):
|
98 |
+
super().__init__()
|
99 |
+
self.nonlinearity = nn.Sequential(
|
100 |
+
nn.LayerNorm(config.hidden_size, config.layer_norm_eps, elementwise_affine=False),
|
101 |
+
nn.Linear(config.hidden_size, config.hidden_size),
|
102 |
+
nn.GELU(),
|
103 |
+
nn.LayerNorm(config.hidden_size, config.layer_norm_eps, elementwise_affine=False),
|
104 |
+
nn.Dropout(config.hidden_dropout_prob),
|
105 |
+
nn.Linear(subword_embedding.size(1), subword_embedding.size(0))
|
106 |
+
)
|
107 |
+
self.initialize(config.hidden_size, subword_embedding)
|
108 |
+
|
109 |
+
def initialize(self, hidden_size, embedding):
|
110 |
+
std = math.sqrt(2.0 / (5.0 * hidden_size))
|
111 |
+
nn.init.trunc_normal_(self.nonlinearity[1].weight, mean=0.0, std=std, a=-2*std, b=2*std)
|
112 |
+
self.nonlinearity[-1].weight = embedding
|
113 |
+
self.nonlinearity[1].bias.data.zero_()
|
114 |
+
self.nonlinearity[-1].bias.data.zero_()
|
115 |
+
|
116 |
+
def forward(self, x, masked_lm_labels=None):
|
117 |
+
if masked_lm_labels is not None:
|
118 |
+
x = torch.index_select(x.flatten(0, 1), 0, torch.nonzero(masked_lm_labels.flatten() != -100).squeeze())
|
119 |
+
x = self.nonlinearity(x)
|
120 |
+
return x
|
121 |
+
|
122 |
+
|
123 |
+
class EncoderLayer(nn.Module):
|
124 |
+
def __init__(self, config):
|
125 |
+
super().__init__()
|
126 |
+
self.attention = Attention(config)
|
127 |
+
self.mlp = FeedForward(config)
|
128 |
+
|
129 |
+
def forward(self, x, padding_mask, relative_embedding):
|
130 |
+
attention_output, attention_probs = self.attention(x, padding_mask, relative_embedding)
|
131 |
+
x = x + attention_output
|
132 |
+
x = x + self.mlp(x)
|
133 |
+
return x, attention_probs
|
134 |
+
|
135 |
+
|
136 |
+
class GeGLU(nn.Module):
|
137 |
+
def forward(self, x):
|
138 |
+
x, gate = x.chunk(2, dim=-1)
|
139 |
+
x = x * gelu_new(gate)
|
140 |
+
return x
|
141 |
+
|
142 |
+
|
143 |
+
class FeedForward(nn.Module):
|
144 |
+
def __init__(self, config):
|
145 |
+
super().__init__()
|
146 |
+
self.mlp = nn.Sequential(
|
147 |
+
nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps, elementwise_affine=False),
|
148 |
+
nn.Linear(config.hidden_size, 2*config.intermediate_size, bias=False),
|
149 |
+
GeGLU(),
|
150 |
+
nn.LayerNorm(config.intermediate_size, eps=config.layer_norm_eps, elementwise_affine=False),
|
151 |
+
nn.Linear(config.intermediate_size, config.hidden_size, bias=False),
|
152 |
+
nn.Dropout(config.hidden_dropout_prob)
|
153 |
+
)
|
154 |
+
self.initialize(config.hidden_size)
|
155 |
+
|
156 |
+
def initialize(self, hidden_size):
|
157 |
+
std = math.sqrt(2.0 / (5.0 * hidden_size))
|
158 |
+
nn.init.trunc_normal_(self.mlp[1].weight, mean=0.0, std=std, a=-2*std, b=2*std)
|
159 |
+
nn.init.trunc_normal_(self.mlp[-2].weight, mean=0.0, std=std, a=-2*std, b=2*std)
|
160 |
+
|
161 |
+
def forward(self, x):
|
162 |
+
return self.mlp(x)
|
163 |
+
|
164 |
+
|
165 |
+
class MaskedSoftmax(torch.autograd.Function):
|
166 |
+
@staticmethod
|
167 |
+
def forward(self, x, mask, dim):
|
168 |
+
self.dim = dim
|
169 |
+
x.masked_fill_(mask, float('-inf'))
|
170 |
+
x = torch.softmax(x, self.dim)
|
171 |
+
x.masked_fill_(mask, 0.0)
|
172 |
+
self.save_for_backward(x)
|
173 |
+
return x
|
174 |
+
|
175 |
+
@staticmethod
|
176 |
+
def backward(self, grad_output):
|
177 |
+
output, = self.saved_tensors
|
178 |
+
input_grad = softmax_backward_data(self, grad_output, output, self.dim, output)
|
179 |
+
return input_grad, None, None
|
180 |
+
|
181 |
+
|
182 |
+
class Attention(nn.Module):
|
183 |
+
def __init__(self, config):
|
184 |
+
super().__init__()
|
185 |
+
|
186 |
+
self.config = config
|
187 |
+
|
188 |
+
if config.hidden_size % config.num_attention_heads != 0:
|
189 |
+
raise ValueError(f"The hidden size {config.hidden_size} is not a multiple of the number of attention heads {config.num_attention_heads}")
|
190 |
+
|
191 |
+
self.hidden_size = config.hidden_size
|
192 |
+
self.num_heads = config.num_attention_heads
|
193 |
+
self.head_size = config.hidden_size // config.num_attention_heads
|
194 |
+
|
195 |
+
self.in_proj_qk = nn.Linear(config.hidden_size, 2*config.hidden_size, bias=True)
|
196 |
+
self.in_proj_v = nn.Linear(config.hidden_size, config.hidden_size, bias=True)
|
197 |
+
self.out_proj = nn.Linear(config.hidden_size, config.hidden_size, bias=True)
|
198 |
+
|
199 |
+
self.pre_layer_norm = nn.LayerNorm(config.hidden_size, config.layer_norm_eps, elementwise_affine=False)
|
200 |
+
self.post_layer_norm = nn.LayerNorm(config.hidden_size, config.layer_norm_eps, elementwise_affine=True)
|
201 |
+
|
202 |
+
position_indices = torch.arange(config.max_position_embeddings, dtype=torch.long).unsqueeze(1) \
|
203 |
+
- torch.arange(config.max_position_embeddings, dtype=torch.long).unsqueeze(0)
|
204 |
+
position_indices = self.make_log_bucket_position(position_indices, config.position_bucket_size, config.max_position_embeddings)
|
205 |
+
position_indices = config.position_bucket_size - 1 + position_indices
|
206 |
+
self.register_buffer("position_indices", position_indices, persistent=True)
|
207 |
+
|
208 |
+
self.dropout = nn.Dropout(config.attention_probs_dropout_prob)
|
209 |
+
self.scale = 1.0 / math.sqrt(3 * self.head_size)
|
210 |
+
self.initialize()
|
211 |
+
|
212 |
+
def make_log_bucket_position(self, relative_pos, bucket_size, max_position):
|
213 |
+
sign = torch.sign(relative_pos)
|
214 |
+
mid = bucket_size // 2
|
215 |
+
abs_pos = torch.where((relative_pos < mid) & (relative_pos > -mid), mid - 1, torch.abs(relative_pos).clamp(max=max_position - 1))
|
216 |
+
log_pos = torch.ceil(torch.log(abs_pos / mid) / math.log((max_position-1) / mid) * (mid - 1)).int() + mid
|
217 |
+
bucket_pos = torch.where(abs_pos <= mid, relative_pos, log_pos * sign).long()
|
218 |
+
return bucket_pos
|
219 |
+
|
220 |
+
def initialize(self):
|
221 |
+
std = math.sqrt(2.0 / (5.0 * self.hidden_size))
|
222 |
+
nn.init.trunc_normal_(self.in_proj_qk.weight, mean=0.0, std=std, a=-2*std, b=2*std)
|
223 |
+
nn.init.trunc_normal_(self.in_proj_v.weight, mean=0.0, std=std, a=-2*std, b=2*std)
|
224 |
+
nn.init.trunc_normal_(self.out_proj.weight, mean=0.0, std=std, a=-2*std, b=2*std)
|
225 |
+
self.in_proj_qk.bias.data.zero_()
|
226 |
+
self.in_proj_v.bias.data.zero_()
|
227 |
+
self.out_proj.bias.data.zero_()
|
228 |
+
|
229 |
+
def compute_attention_scores(self, hidden_states, relative_embedding):
|
230 |
+
key_len, batch_size, _ = hidden_states.size()
|
231 |
+
query_len = key_len
|
232 |
+
|
233 |
+
if self.position_indices.size(0) < query_len:
|
234 |
+
position_indices = torch.arange(query_len, dtype=torch.long).unsqueeze(1) \
|
235 |
+
- torch.arange(query_len, dtype=torch.long).unsqueeze(0)
|
236 |
+
position_indices = self.make_log_bucket_position(position_indices, self.position_bucket_size, 512)
|
237 |
+
position_indices = self.position_bucket_size - 1 + position_indices
|
238 |
+
self.position_indices = position_indices.to(hidden_states.device)
|
239 |
+
|
240 |
+
hidden_states = self.pre_layer_norm(hidden_states)
|
241 |
+
|
242 |
+
query, key = self.in_proj_qk(hidden_states).chunk(2, dim=2) # shape: [T, B, D]
|
243 |
+
value = self.in_proj_v(hidden_states) # shape: [T, B, D]
|
244 |
+
|
245 |
+
query = query.reshape(query_len, batch_size * self.num_heads, self.head_size).transpose(0, 1)
|
246 |
+
key = key.reshape(key_len, batch_size * self.num_heads, self.head_size).transpose(0, 1)
|
247 |
+
value = value.view(key_len, batch_size * self.num_heads, self.head_size).transpose(0, 1)
|
248 |
+
|
249 |
+
attention_scores = torch.bmm(query, key.transpose(1, 2) * self.scale)
|
250 |
+
|
251 |
+
query_pos, key_pos = self.in_proj_qk(self.dropout(relative_embedding)).chunk(2, dim=-1) # shape: [2T-1, D]
|
252 |
+
query_pos = query_pos.view(-1, self.num_heads, self.head_size) # shape: [2T-1, H, D]
|
253 |
+
key_pos = key_pos.view(-1, self.num_heads, self.head_size) # shape: [2T-1, H, D]
|
254 |
+
|
255 |
+
query = query.view(batch_size, self.num_heads, query_len, self.head_size)
|
256 |
+
key = key.view(batch_size, self.num_heads, query_len, self.head_size)
|
257 |
+
|
258 |
+
attention_c_p = torch.einsum("bhqd,khd->bhqk", query, key_pos.squeeze(1) * self.scale)
|
259 |
+
attention_p_c = torch.einsum("bhkd,qhd->bhqk", key * self.scale, query_pos.squeeze(1))
|
260 |
+
|
261 |
+
position_indices = self.position_indices[:query_len, :key_len].expand(batch_size, self.num_heads, -1, -1)
|
262 |
+
attention_c_p = attention_c_p.gather(3, position_indices)
|
263 |
+
attention_p_c = attention_p_c.gather(2, position_indices)
|
264 |
+
|
265 |
+
attention_scores = attention_scores.view(batch_size, self.num_heads, query_len, key_len)
|
266 |
+
attention_scores.add_(attention_c_p)
|
267 |
+
attention_scores.add_(attention_p_c)
|
268 |
+
|
269 |
+
return attention_scores, value
|
270 |
+
|
271 |
+
def compute_output(self, attention_probs, value):
|
272 |
+
attention_probs = self.dropout(attention_probs)
|
273 |
+
context = torch.bmm(attention_probs.flatten(0, 1), value) # shape: [B*H, Q, D]
|
274 |
+
context = context.transpose(0, 1).reshape(context.size(1), -1, self.hidden_size) # shape: [Q, B, H*D]
|
275 |
+
context = self.out_proj(context)
|
276 |
+
context = self.post_layer_norm(context)
|
277 |
+
context = self.dropout(context)
|
278 |
+
return context
|
279 |
+
|
280 |
+
def forward(self, hidden_states, attention_mask, relative_embedding):
|
281 |
+
attention_scores, value = self.compute_attention_scores(hidden_states, relative_embedding)
|
282 |
+
attention_probs = MaskedSoftmax.apply(attention_scores, attention_mask, -1)
|
283 |
+
return self.compute_output(attention_probs, value), attention_probs.detach()
|
284 |
+
|
285 |
+
|
286 |
+
class Embedding(nn.Module):
|
287 |
+
def __init__(self, config):
|
288 |
+
super().__init__()
|
289 |
+
self.hidden_size = config.hidden_size
|
290 |
+
|
291 |
+
self.word_embedding = nn.Embedding(config.vocab_size, config.hidden_size)
|
292 |
+
self.word_layer_norm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps, elementwise_affine=False)
|
293 |
+
self.dropout = nn.Dropout(config.hidden_dropout_prob)
|
294 |
+
|
295 |
+
self.relative_embedding = nn.Parameter(torch.empty(2 * config.position_bucket_size - 1, config.hidden_size))
|
296 |
+
self.relative_layer_norm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
|
297 |
+
|
298 |
+
self.initialize()
|
299 |
+
|
300 |
+
def initialize(self):
|
301 |
+
std = math.sqrt(2.0 / (5.0 * self.hidden_size))
|
302 |
+
nn.init.trunc_normal_(self.relative_embedding, mean=0.0, std=std, a=-2*std, b=2*std)
|
303 |
+
nn.init.trunc_normal_(self.word_embedding.weight, mean=0.0, std=std, a=-2*std, b=2*std)
|
304 |
+
|
305 |
+
def forward(self, input_ids):
|
306 |
+
word_embedding = self.dropout(self.word_layer_norm(self.word_embedding(input_ids)))
|
307 |
+
relative_embeddings = self.relative_layer_norm(self.relative_embedding)
|
308 |
+
return word_embedding, relative_embeddings
|
309 |
+
|
310 |
+
|
311 |
+
#
|
312 |
+
# HuggingFace wrappers
|
313 |
+
#
|
314 |
+
|
315 |
+
class LtgBertPreTrainedModel(PreTrainedModel):
|
316 |
+
"""
|
317 |
+
An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
|
318 |
+
models.
|
319 |
+
"""
|
320 |
+
|
321 |
+
config_class = LtgBertConfig
|
322 |
+
base_model_prefix = "bnc-bert"
|
323 |
+
supports_gradient_checkpointing = True
|
324 |
+
|
325 |
+
def _set_gradient_checkpointing(self, module, value=False):
|
326 |
+
if isinstance(module, Encoder):
|
327 |
+
module.activation_checkpointing = value
|
328 |
+
|
329 |
+
def _init_weights(self, _):
|
330 |
+
pass # everything is already initialized
|
331 |
+
|
332 |
+
|
333 |
+
LTG_BERT_START_DOCSTRING = r"""
|
334 |
+
This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
|
335 |
+
library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
|
336 |
+
etc.)
|
337 |
+
This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
|
338 |
+
Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
|
339 |
+
and behavior.
|
340 |
+
Parameters:
|
341 |
+
config ([`LtgBertConfig`]): Model configuration class with all the parameters of the model.
|
342 |
+
Initializing with a config file does not load the weights associated with the model, only the
|
343 |
+
configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
|
344 |
+
"""
|
345 |
+
|
346 |
+
LTG_BERT_INPUTS_DOCSTRING = r"""
|
347 |
+
Args:
|
348 |
+
input_ids (`torch.LongTensor` of shape `({0})`):
|
349 |
+
Indices of input sequence tokens in the vocabulary.
|
350 |
+
Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
|
351 |
+
[`PreTrainedTokenizer.__call__`] for details.
|
352 |
+
[What are input IDs?](../glossary#input-ids)
|
353 |
+
attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*):
|
354 |
+
Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
|
355 |
+
- 1 for tokens that are **not masked**,
|
356 |
+
- 0 for tokens that are **masked**.
|
357 |
+
[What are attention masks?](../glossary#attention-mask)
|
358 |
+
output_hidden_states (`bool`, *optional*):
|
359 |
+
Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
|
360 |
+
more detail.
|
361 |
+
output_attentions (`bool`, *optional*):
|
362 |
+
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
|
363 |
+
tensors for more detail.
|
364 |
+
return_dict (`bool`, *optional*):
|
365 |
+
Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
|
366 |
+
"""
|
367 |
+
|
368 |
+
|
369 |
+
@add_start_docstrings(
|
370 |
+
"The bare LTG-BERT transformer outputting raw hidden-states without any specific head on top.",
|
371 |
+
LTG_BERT_START_DOCSTRING,
|
372 |
+
)
|
373 |
+
class LtgBertModel(LtgBertPreTrainedModel):
|
374 |
+
def __init__(self, config, add_mlm_layer=False):
|
375 |
+
super().__init__(config)
|
376 |
+
self.config = config
|
377 |
+
|
378 |
+
self.embedding = Embedding(config)
|
379 |
+
self.transformer = Encoder(config, activation_checkpointing=False)
|
380 |
+
self.classifier = MaskClassifier(config, self.embedding.word_embedding.weight) if add_mlm_layer else None
|
381 |
+
|
382 |
+
def get_input_embeddings(self):
|
383 |
+
return self.embedding.word_embedding
|
384 |
+
|
385 |
+
def set_input_embeddings(self, value):
|
386 |
+
self.embedding.word_embedding = value
|
387 |
+
|
388 |
+
def get_contextualized_embeddings(
|
389 |
+
self,
|
390 |
+
input_ids: Optional[torch.Tensor] = None,
|
391 |
+
attention_mask: Optional[torch.Tensor] = None
|
392 |
+
) -> List[torch.Tensor]:
|
393 |
+
if input_ids is not None:
|
394 |
+
input_shape = input_ids.size()
|
395 |
+
else:
|
396 |
+
raise ValueError("You have to specify input_ids")
|
397 |
+
|
398 |
+
batch_size, seq_length = input_shape
|
399 |
+
device = input_ids.device
|
400 |
+
|
401 |
+
if attention_mask is None:
|
402 |
+
attention_mask = torch.zeros(batch_size, seq_length, dtype=torch.bool, device=device)
|
403 |
+
else:
|
404 |
+
attention_mask = ~attention_mask.bool()
|
405 |
+
attention_mask = attention_mask.unsqueeze(1).unsqueeze(2)
|
406 |
+
|
407 |
+
static_embeddings, relative_embedding = self.embedding(input_ids.t())
|
408 |
+
contextualized_embeddings, attention_probs = self.transformer(static_embeddings, attention_mask, relative_embedding)
|
409 |
+
contextualized_embeddings = [e.transpose(0, 1) for e in contextualized_embeddings]
|
410 |
+
last_layer = contextualized_embeddings[-1]
|
411 |
+
contextualized_embeddings = [contextualized_embeddings[0]] + [
|
412 |
+
contextualized_embeddings[i] - contextualized_embeddings[i - 1]
|
413 |
+
for i in range(1, len(contextualized_embeddings))
|
414 |
+
]
|
415 |
+
return last_layer, contextualized_embeddings, attention_probs
|
416 |
+
|
417 |
+
@add_start_docstrings_to_model_forward(LTG_BERT_INPUTS_DOCSTRING.format("batch_size, sequence_length"))
|
418 |
+
def forward(
|
419 |
+
self,
|
420 |
+
input_ids: Optional[torch.Tensor] = None,
|
421 |
+
attention_mask: Optional[torch.Tensor] = None,
|
422 |
+
output_hidden_states: Optional[bool] = None,
|
423 |
+
output_attentions: Optional[bool] = None,
|
424 |
+
return_dict: Optional[bool] = None,
|
425 |
+
token_type_ids=None
|
426 |
+
) -> Union[Tuple[torch.Tensor], BaseModelOutput]:
|
427 |
+
|
428 |
+
output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
|
429 |
+
output_hidden_states = (
|
430 |
+
output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
|
431 |
+
)
|
432 |
+
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
|
433 |
+
|
434 |
+
sequence_output, contextualized_embeddings, attention_probs = self.get_contextualized_embeddings(input_ids, attention_mask)
|
435 |
+
|
436 |
+
if not return_dict:
|
437 |
+
return (
|
438 |
+
sequence_output,
|
439 |
+
*([contextualized_embeddings] if output_hidden_states else []),
|
440 |
+
*([attention_probs] if output_attentions else [])
|
441 |
+
)
|
442 |
+
|
443 |
+
return BaseModelOutput(
|
444 |
+
last_hidden_state=sequence_output,
|
445 |
+
hidden_states=contextualized_embeddings if output_hidden_states else None,
|
446 |
+
attentions=attention_probs if output_attentions else None
|
447 |
+
)
|
448 |
+
|
449 |
+
|
450 |
+
@add_start_docstrings("""LTG-BERT model with a `language modeling` head on top.""", LTG_BERT_START_DOCSTRING)
|
451 |
+
class LtgBertForMaskedLM(LtgBertModel):
|
452 |
+
_keys_to_ignore_on_load_unexpected = ["head"]
|
453 |
+
|
454 |
+
def __init__(self, config):
|
455 |
+
super().__init__(config, add_mlm_layer=True)
|
456 |
+
|
457 |
+
def get_output_embeddings(self):
|
458 |
+
return self.classifier.nonlinearity[-1].weight
|
459 |
+
|
460 |
+
def set_output_embeddings(self, new_embeddings):
|
461 |
+
self.classifier.nonlinearity[-1].weight = new_embeddings
|
462 |
+
|
463 |
+
@add_start_docstrings_to_model_forward(LTG_BERT_INPUTS_DOCSTRING.format("batch_size, sequence_length"))
|
464 |
+
def forward(
|
465 |
+
self,
|
466 |
+
input_ids: Optional[torch.Tensor] = None,
|
467 |
+
attention_mask: Optional[torch.Tensor] = None,
|
468 |
+
output_hidden_states: Optional[bool] = None,
|
469 |
+
output_attentions: Optional[bool] = None,
|
470 |
+
return_dict: Optional[bool] = None,
|
471 |
+
labels: Optional[torch.LongTensor] = None,
|
472 |
+
token_type_ids=None
|
473 |
+
) -> Union[Tuple[torch.Tensor], MaskedLMOutput]:
|
474 |
+
r"""
|
475 |
+
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
|
476 |
+
Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
|
477 |
+
config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
|
478 |
+
loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
|
479 |
+
"""
|
480 |
+
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
|
481 |
+
|
482 |
+
sequence_output, contextualized_embeddings, attention_probs = self.get_contextualized_embeddings(input_ids, attention_mask)
|
483 |
+
subword_prediction = self.classifier(sequence_output)
|
484 |
+
|
485 |
+
masked_lm_loss = None
|
486 |
+
if labels is not None:
|
487 |
+
masked_lm_loss = F.cross_entropy(subword_prediction.flatten(0, 1), labels.flatten())
|
488 |
+
|
489 |
+
if not return_dict:
|
490 |
+
output = (
|
491 |
+
subword_prediction,
|
492 |
+
*([contextualized_embeddings] if output_hidden_states else []),
|
493 |
+
*([attention_probs] if output_attentions else [])
|
494 |
+
)
|
495 |
+
return ((masked_lm_loss,) + output) if masked_lm_loss is not None else output
|
496 |
+
|
497 |
+
return MaskedLMOutput(
|
498 |
+
loss=masked_lm_loss,
|
499 |
+
logits=subword_prediction,
|
500 |
+
hidden_states=contextualized_embeddings if output_hidden_states else None,
|
501 |
+
attentions=attention_probs if output_attentions else None
|
502 |
+
)
|
503 |
+
|
504 |
+
|
505 |
+
class Classifier(nn.Module):
|
506 |
+
def __init__(self, config, num_labels: int):
|
507 |
+
super().__init__()
|
508 |
+
|
509 |
+
drop_out = getattr(config, "classifier_dropout", config.hidden_dropout_prob)
|
510 |
+
|
511 |
+
self.nonlinearity = nn.Sequential(
|
512 |
+
nn.LayerNorm(config.hidden_size, config.layer_norm_eps, elementwise_affine=False),
|
513 |
+
nn.Linear(config.hidden_size, config.hidden_size),
|
514 |
+
nn.GELU(),
|
515 |
+
nn.LayerNorm(config.hidden_size, config.layer_norm_eps, elementwise_affine=False),
|
516 |
+
nn.Dropout(drop_out),
|
517 |
+
nn.Linear(config.hidden_size, num_labels)
|
518 |
+
)
|
519 |
+
self.initialize(config.hidden_size)
|
520 |
+
|
521 |
+
def initialize(self, hidden_size):
|
522 |
+
std = math.sqrt(2.0 / (5.0 * hidden_size))
|
523 |
+
nn.init.trunc_normal_(self.nonlinearity[1].weight, mean=0.0, std=std, a=-2*std, b=2*std)
|
524 |
+
nn.init.trunc_normal_(self.nonlinearity[-1].weight, mean=0.0, std=std, a=-2*std, b=2*std)
|
525 |
+
self.nonlinearity[1].bias.data.zero_()
|
526 |
+
self.nonlinearity[-1].bias.data.zero_()
|
527 |
+
|
528 |
+
def forward(self, x):
|
529 |
+
x = self.nonlinearity(x)
|
530 |
+
return x
|
531 |
+
|
532 |
+
|
533 |
+
@add_start_docstrings(
|
534 |
+
"""
|
535 |
+
LTG-BERT model with a sequence classification/regression head on top (a linear layer on top of the pooled
|
536 |
+
output) e.g. for GLUE tasks.
|
537 |
+
""",
|
538 |
+
LTG_BERT_START_DOCSTRING,
|
539 |
+
)
|
540 |
+
class LtgBertForSequenceClassification(LtgBertModel):
|
541 |
+
_keys_to_ignore_on_load_unexpected = ["classifier"]
|
542 |
+
_keys_to_ignore_on_load_missing = ["head"]
|
543 |
+
|
544 |
+
def __init__(self, config):
|
545 |
+
super().__init__(config, add_mlm_layer=False)
|
546 |
+
|
547 |
+
self.num_labels = config.num_labels
|
548 |
+
self.head = Classifier(config, self.num_labels)
|
549 |
+
|
550 |
+
@add_start_docstrings_to_model_forward(LTG_BERT_INPUTS_DOCSTRING.format("batch_size, sequence_length"))
|
551 |
+
def forward(
|
552 |
+
self,
|
553 |
+
input_ids: Optional[torch.Tensor] = None,
|
554 |
+
attention_mask: Optional[torch.Tensor] = None,
|
555 |
+
output_attentions: Optional[bool] = None,
|
556 |
+
output_hidden_states: Optional[bool] = None,
|
557 |
+
return_dict: Optional[bool] = None,
|
558 |
+
labels: Optional[torch.LongTensor] = None,
|
559 |
+
) -> Union[Tuple[torch.Tensor], SequenceClassifierOutput]:
|
560 |
+
r"""
|
561 |
+
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
|
562 |
+
Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
|
563 |
+
config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
|
564 |
+
`config.num_labels > 1` a classification loss is computed (Cross-Entropy).
|
565 |
+
"""
|
566 |
+
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
|
567 |
+
|
568 |
+
sequence_output, contextualized_embeddings, attention_probs = self.get_contextualized_embeddings(input_ids, attention_mask)
|
569 |
+
logits = self.head(sequence_output[:, 0, :])
|
570 |
+
|
571 |
+
loss = None
|
572 |
+
if labels is not None:
|
573 |
+
if self.config.problem_type is None:
|
574 |
+
if self.num_labels == 1:
|
575 |
+
self.config.problem_type = "regression"
|
576 |
+
elif self.num_labels > 1 and (labels.dtype == torch.long or labels.dtype == torch.int):
|
577 |
+
self.config.problem_type = "single_label_classification"
|
578 |
+
else:
|
579 |
+
self.config.problem_type = "multi_label_classification"
|
580 |
+
|
581 |
+
if self.config.problem_type == "regression":
|
582 |
+
loss_fct = nn.MSELoss()
|
583 |
+
if self.num_labels == 1:
|
584 |
+
loss = loss_fct(logits.squeeze(), labels.squeeze())
|
585 |
+
else:
|
586 |
+
loss = loss_fct(logits, labels)
|
587 |
+
elif self.config.problem_type == "single_label_classification":
|
588 |
+
loss_fct = nn.CrossEntropyLoss()
|
589 |
+
loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1))
|
590 |
+
elif self.config.problem_type == "multi_label_classification":
|
591 |
+
loss_fct = nn.BCEWithLogitsLoss()
|
592 |
+
loss = loss_fct(logits, labels)
|
593 |
+
|
594 |
+
if not return_dict:
|
595 |
+
output = (
|
596 |
+
logits,
|
597 |
+
*([contextualized_embeddings] if output_hidden_states else []),
|
598 |
+
*([attention_probs] if output_attentions else [])
|
599 |
+
)
|
600 |
+
return ((loss,) + output) if loss is not None else output
|
601 |
+
|
602 |
+
return SequenceClassifierOutput(
|
603 |
+
loss=loss,
|
604 |
+
logits=logits,
|
605 |
+
hidden_states=contextualized_embeddings if output_hidden_states else None,
|
606 |
+
attentions=attention_probs if output_attentions else None
|
607 |
+
)
|
608 |
+
|
609 |
+
|
610 |
+
@add_start_docstrings(
|
611 |
+
"""
|
612 |
+
LTG-BERT model with a token classification head on top (a linear layer on top of the hidden-states output) e.g. for
|
613 |
+
Named-Entity-Recognition (NER) tasks.
|
614 |
+
""",
|
615 |
+
LTG_BERT_START_DOCSTRING,
|
616 |
+
)
|
617 |
+
class LtgBertForTokenClassification(LtgBertModel):
|
618 |
+
_keys_to_ignore_on_load_unexpected = ["classifier"]
|
619 |
+
_keys_to_ignore_on_load_missing = ["head"]
|
620 |
+
|
621 |
+
def __init__(self, config):
|
622 |
+
super().__init__(config, add_mlm_layer=False)
|
623 |
+
|
624 |
+
self.num_labels = config.num_labels
|
625 |
+
self.head = Classifier(config, self.num_labels)
|
626 |
+
|
627 |
+
@add_start_docstrings_to_model_forward(LTG_BERT_INPUTS_DOCSTRING.format("batch_size, sequence_length"))
|
628 |
+
def forward(
|
629 |
+
self,
|
630 |
+
input_ids: Optional[torch.Tensor] = None,
|
631 |
+
attention_mask: Optional[torch.Tensor] = None,
|
632 |
+
token_type_ids: Optional[torch.Tensor] = None,
|
633 |
+
position_ids: Optional[torch.Tensor] = None,
|
634 |
+
output_attentions: Optional[bool] = None,
|
635 |
+
output_hidden_states: Optional[bool] = None,
|
636 |
+
return_dict: Optional[bool] = None,
|
637 |
+
labels: Optional[torch.LongTensor] = None,
|
638 |
+
) -> Union[Tuple[torch.Tensor], TokenClassifierOutput]:
|
639 |
+
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
|
640 |
+
|
641 |
+
sequence_output, contextualized_embeddings, attention_probs = self.get_contextualized_embeddings(input_ids, attention_mask)
|
642 |
+
logits = self.head(sequence_output)
|
643 |
+
|
644 |
+
loss = None
|
645 |
+
if labels is not None:
|
646 |
+
loss_fct = nn.CrossEntropyLoss()
|
647 |
+
loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1))
|
648 |
+
|
649 |
+
if not return_dict:
|
650 |
+
output = (
|
651 |
+
logits,
|
652 |
+
*([contextualized_embeddings] if output_hidden_states else []),
|
653 |
+
*([attention_probs] if output_attentions else [])
|
654 |
+
)
|
655 |
+
return ((loss,) + output) if loss is not None else output
|
656 |
+
|
657 |
+
return TokenClassifierOutput(
|
658 |
+
loss=loss,
|
659 |
+
logits=logits,
|
660 |
+
hidden_states=contextualized_embeddings if output_hidden_states else None,
|
661 |
+
attentions=attention_probs if output_attentions else None
|
662 |
+
)
|
663 |
+
|
664 |
+
|
665 |
+
@add_start_docstrings(
|
666 |
+
"""
|
667 |
+
LTG-BERT model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear
|
668 |
+
layers on top of the hidden-states output to compute `span start logits` and `span end logits`).
|
669 |
+
""",
|
670 |
+
LTG_BERT_START_DOCSTRING,
|
671 |
+
)
|
672 |
+
class LtgBertForQuestionAnswering(LtgBertModel):
|
673 |
+
_keys_to_ignore_on_load_unexpected = ["classifier"]
|
674 |
+
_keys_to_ignore_on_load_missing = ["head"]
|
675 |
+
|
676 |
+
def __init__(self, config):
|
677 |
+
super().__init__(config, add_mlm_layer=False)
|
678 |
+
|
679 |
+
self.num_labels = config.num_labels
|
680 |
+
self.head = Classifier(config, self.num_labels)
|
681 |
+
|
682 |
+
@add_start_docstrings_to_model_forward(LTG_BERT_INPUTS_DOCSTRING.format("batch_size, sequence_length"))
|
683 |
+
def forward(
|
684 |
+
self,
|
685 |
+
input_ids: Optional[torch.Tensor] = None,
|
686 |
+
attention_mask: Optional[torch.Tensor] = None,
|
687 |
+
token_type_ids: Optional[torch.Tensor] = None,
|
688 |
+
position_ids: Optional[torch.Tensor] = None,
|
689 |
+
output_attentions: Optional[bool] = None,
|
690 |
+
output_hidden_states: Optional[bool] = None,
|
691 |
+
return_dict: Optional[bool] = None,
|
692 |
+
start_positions: Optional[torch.Tensor] = None,
|
693 |
+
end_positions: Optional[torch.Tensor] = None
|
694 |
+
) -> Union[Tuple[torch.Tensor], QuestionAnsweringModelOutput]:
|
695 |
+
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
|
696 |
+
|
697 |
+
sequence_output, contextualized_embeddings, attention_probs = self.get_contextualized_embeddings(input_ids, attention_mask)
|
698 |
+
logits = self.head(sequence_output)
|
699 |
+
|
700 |
+
start_logits, end_logits = logits.split(1, dim=-1)
|
701 |
+
start_logits = start_logits.squeeze(-1).contiguous()
|
702 |
+
end_logits = end_logits.squeeze(-1).contiguous()
|
703 |
+
|
704 |
+
total_loss = None
|
705 |
+
if start_positions is not None and end_positions is not None:
|
706 |
+
# If we are on multi-GPU, split add a dimension
|
707 |
+
if len(start_positions.size()) > 1:
|
708 |
+
start_positions = start_positions.squeeze(-1)
|
709 |
+
if len(end_positions.size()) > 1:
|
710 |
+
end_positions = end_positions.squeeze(-1)
|
711 |
+
|
712 |
+
# sometimes the start/end positions are outside our model inputs, we ignore these terms
|
713 |
+
ignored_index = start_logits.size(1)
|
714 |
+
start_positions = start_positions.clamp(0, ignored_index)
|
715 |
+
end_positions = end_positions.clamp(0, ignored_index)
|
716 |
+
|
717 |
+
loss_fct = nn.CrossEntropyLoss(ignore_index=ignored_index)
|
718 |
+
start_loss = loss_fct(start_logits, start_positions)
|
719 |
+
end_loss = loss_fct(end_logits, end_positions)
|
720 |
+
total_loss = (start_loss + end_loss) / 2
|
721 |
+
|
722 |
+
if not return_dict:
|
723 |
+
output = (
|
724 |
+
start_logits,
|
725 |
+
end_logits,
|
726 |
+
*([contextualized_embeddings] if output_hidden_states else []),
|
727 |
+
*([attention_probs] if output_attentions else [])
|
728 |
+
)
|
729 |
+
return ((total_loss,) + output) if total_loss is not None else output
|
730 |
+
|
731 |
+
return QuestionAnsweringModelOutput(
|
732 |
+
loss=total_loss,
|
733 |
+
start_logits=start_logits,
|
734 |
+
end_logits=end_logits,
|
735 |
+
hidden_states=contextualized_embeddings if output_hidden_states else None,
|
736 |
+
attentions=attention_probs if output_attentions else None
|
737 |
+
)
|
738 |
+
|
739 |
+
|
740 |
+
@add_start_docstrings(
|
741 |
+
"""
|
742 |
+
LTG-BERT model with a multiple choice classification head on top (a linear layer on top of the pooled output and a
|
743 |
+
softmax) e.g. for RocStories/SWAG tasks.
|
744 |
+
""",
|
745 |
+
LTG_BERT_START_DOCSTRING,
|
746 |
+
)
|
747 |
+
class LtgBertForMultipleChoice(LtgBertModel):
|
748 |
+
_keys_to_ignore_on_load_unexpected = ["classifier"]
|
749 |
+
_keys_to_ignore_on_load_missing = ["head"]
|
750 |
+
|
751 |
+
def __init__(self, config):
|
752 |
+
super().__init__(config, add_mlm_layer=False)
|
753 |
+
|
754 |
+
self.num_labels = getattr(config, "num_labels", 2)
|
755 |
+
self.head = Classifier(config, self.num_labels)
|
756 |
+
|
757 |
+
@add_start_docstrings_to_model_forward(LTG_BERT_INPUTS_DOCSTRING.format("batch_size, num_choices, sequence_length"))
|
758 |
+
def forward(
|
759 |
+
self,
|
760 |
+
input_ids: Optional[torch.Tensor] = None,
|
761 |
+
attention_mask: Optional[torch.Tensor] = None,
|
762 |
+
token_type_ids: Optional[torch.Tensor] = None,
|
763 |
+
position_ids: Optional[torch.Tensor] = None,
|
764 |
+
labels: Optional[torch.Tensor] = None,
|
765 |
+
output_attentions: Optional[bool] = None,
|
766 |
+
output_hidden_states: Optional[bool] = None,
|
767 |
+
return_dict: Optional[bool] = None
|
768 |
+
) -> Union[Tuple[torch.Tensor], MultipleChoiceModelOutput]:
|
769 |
+
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
|
770 |
+
num_choices = input_ids.shape[1]
|
771 |
+
|
772 |
+
flat_input_ids = input_ids.view(-1, input_ids.size(-1))
|
773 |
+
flat_attention_mask = attention_mask.view(-1, attention_mask.size(-1)) if attention_mask is not None else None
|
774 |
+
|
775 |
+
sequence_output, contextualized_embeddings, attention_probs = self.get_contextualized_embeddings(flat_input_ids, flat_attention_mask)
|
776 |
+
logits = self.head(sequence_output)
|
777 |
+
reshaped_logits = logits.view(-1, num_choices)
|
778 |
+
|
779 |
+
loss = None
|
780 |
+
if labels is not None:
|
781 |
+
loss_fct = nn.CrossEntropyLoss()
|
782 |
+
loss = loss_fct(reshaped_logits, labels)
|
783 |
+
|
784 |
+
if not return_dict:
|
785 |
+
output = (
|
786 |
+
reshaped_logits,
|
787 |
+
*([contextualized_embeddings] if output_hidden_states else []),
|
788 |
+
*([attention_probs] if output_attentions else [])
|
789 |
+
)
|
790 |
+
return ((loss,) + output) if loss is not None else output
|
791 |
+
|
792 |
+
return MultipleChoiceModelOutput(
|
793 |
+
loss=loss,
|
794 |
+
logits=reshaped_logits,
|
795 |
+
hidden_states=contextualized_embeddings if output_hidden_states else None,
|
796 |
+
attentions=attention_probs if output_attentions else None
|
797 |
+
)
|
pytorch_model.bin
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:3064d81b3099067547da5d9f4eaba6c12e6c17031d8963e85e168006bc8f28e2
|
3 |
+
size 136182801
|
special_tokens_map.json
ADDED
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"bos_token": "[BOS]",
|
3 |
+
"cls_token": "[CLS]",
|
4 |
+
"eos_token": "[EOS]",
|
5 |
+
"mask_token": "[MASK]",
|
6 |
+
"pad_token": "[PAD]",
|
7 |
+
"sep_token": "[SEP]",
|
8 |
+
"unk_token": "[UNK]"
|
9 |
+
}
|
tokenizer.json
ADDED
The diff for this file is too large to render.
See raw diff
|
|
tokenizer_config.json
ADDED
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"model_max_length": 1000000000000000019884624838656,
|
3 |
+
"tokenizer_class": "PreTrainedTokenizerFast"
|
4 |
+
}
|