ltgoslo commited on
Commit
ece3ae2
1 Parent(s): 2daed62

Upload folder using huggingface_hub

Browse files
README.md ADDED
@@ -0,0 +1,69 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ language:
3
+ - kk
4
+ inference: false
5
+ tags:
6
+ - BERT
7
+ - HPLT
8
+ - encoder
9
+ license: apache-2.0
10
+ datasets:
11
+ - HPLT/hplt_monolingual_v1_2
12
+ ---
13
+
14
+ # HPLT Bert for Kazakh
15
+
16
+ <img src="https://hplt-project.org/_next/static/media/logo-hplt.d5e16ca5.svg" width=12.5%>
17
+
18
+ This is one of the encoder-only monolingual language models trained as a first release by the [HPLT project](https://hplt-project.org/).
19
+ It is a so called masked language models. In particular, we used the modification of the classic BERT model named [LTG-BERT](https://aclanthology.org/2023.findings-eacl.146/).
20
+
21
+ A monolingual LTG-BERT model is trained for every major language in the [HPLT 1.2 data release](https://hplt-project.org/datasets/v1.2) (*75* models total).
22
+
23
+ All the HPLT encoder-only models use the same hyper-parameters, roughly following the BERT-base setup:
24
+ - hidden size: 768
25
+ - attention heads: 12
26
+ - layers: 12
27
+ - vocabulary size: 32768
28
+
29
+ Every model uses its own tokenizer trained on language-specific HPLT data.
30
+ See sizes of the training corpora, evaluation results and more in our [language model training report](https://hplt-project.org/HPLT_D4_1___First_language_models_trained.pdf).
31
+
32
+ [The training code](https://github.com/hplt-project/HPLT-WP4).
33
+
34
+ [The training statistics of all 75 runs](https://api.wandb.ai/links/ltg/kduj7mjn)
35
+
36
+ ## Example usage
37
+
38
+ This model currently needs a custom wrapper from `modeling_ltgbert.py`, you should therefore load the model with `trust_remote_code=True`.
39
+
40
+ ```python
41
+ import torch
42
+ from transformers import AutoTokenizer, AutoModelForMaskedLM
43
+
44
+ tokenizer = AutoTokenizer.from_pretrained("HPLT/hplt_bert_base_en")
45
+ model = AutoModelForMaskedLM.from_pretrained("HPLT/hplt_bert_base_en", trust_remote_code=True)
46
+
47
+ mask_id = tokenizer.convert_tokens_to_ids("[MASK]")
48
+ input_text = tokenizer("It's a beautiful[MASK].", return_tensors="pt")
49
+ output_p = model(**input_text)
50
+ output_text = torch.where(input_text.input_ids == mask_id, output_p.logits.argmax(-1), input_text.input_ids)
51
+
52
+ # should output: '[CLS] It's a beautiful place.[SEP]'
53
+ print(tokenizer.decode(output_text[0].tolist()))
54
+ ```
55
+
56
+ The following classes are currently implemented: `AutoModel`, `AutoModelMaskedLM`, `AutoModelForSequenceClassification`, `AutoModelForTokenClassification`, `AutoModelForQuestionAnswering` and `AutoModeltForMultipleChoice`.
57
+
58
+ ## Cite us
59
+
60
+ ```bibtex
61
+ @misc{degibert2024new,
62
+ title={A New Massive Multilingual Dataset for High-Performance Language Technologies},
63
+ author={Ona de Gibert and Graeme Nail and Nikolay Arefyev and Marta Bañón and Jelmer van der Linde and Shaoxiong Ji and Jaume Zaragoza-Bernabeu and Mikko Aulamo and Gema Ramírez-Sánchez and Andrey Kutuzov and Sampo Pyysalo and Stephan Oepen and Jörg Tiedemann},
64
+ year={2024},
65
+ eprint={2403.14009},
66
+ archivePrefix={arXiv},
67
+ primaryClass={cs.CL}
68
+ }
69
+ ```
__init__.py ADDED
File without changes
config.json ADDED
@@ -0,0 +1,25 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "LtgbertForMaskedLM"
4
+ ],
5
+ "auto_map": {
6
+ "AutoConfig": "configuration_ltgbert.LtgbertConfig",
7
+ "AutoModel": "modeling_ltgbert.LtgbertModel",
8
+ "AutoModelForMaskedLM": "modeling_ltgbert.LtgbertForMaskedLM",
9
+ "AutoModelForSequenceClassification": "modeling_ltgbert.LtgbertForSequenceClassification",
10
+ "AutoModelForTokenClassification": "modeling_ltgbert.LtgbertForTokenClassification",
11
+ "AutoModelForQuestionAnswering": "modeling_ltgbert.LtgbertForQuestionAnswering",
12
+ "AutoModelForMultipleChoice": "modeling_ltgbert.LtgbertForMultipleChoice"
13
+ },
14
+ "attention_probs_dropout_prob": 0.1,
15
+ "hidden_dropout_prob": 0.1,
16
+ "hidden_size": 768,
17
+ "intermediate_size": 2560,
18
+ "layer_norm_eps": 1e-07,
19
+ "max_position_embeddings": 512,
20
+ "num_attention_heads": 12,
21
+ "num_hidden_layers": 12,
22
+ "position_bucket_size": 32,
23
+ "torch_dtype": "float32",
24
+ "vocab_size": 32768
25
+ }
configuration_ltgbert.py ADDED
@@ -0,0 +1,34 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from transformers.configuration_utils import PretrainedConfig
2
+
3
+
4
+ class LtgbertConfig(PretrainedConfig):
5
+ """Configuration class to store the configuration of a `LtgbertModel`.
6
+ """
7
+ def __init__(
8
+ self,
9
+ vocab_size=32768,
10
+ attention_probs_dropout_prob=0.1,
11
+ hidden_dropout_prob=0.1,
12
+ hidden_size=768,
13
+ intermediate_size=2048,
14
+ max_position_embeddings=512,
15
+ position_bucket_size=32,
16
+ num_attention_heads=12,
17
+ num_hidden_layers=12,
18
+ layer_norm_eps=1.0e-7,
19
+ output_all_encoded_layers=True,
20
+ **kwargs,
21
+ ):
22
+ super().__init__(**kwargs)
23
+
24
+ self.vocab_size = vocab_size
25
+ self.hidden_size = hidden_size
26
+ self.num_hidden_layers = num_hidden_layers
27
+ self.num_attention_heads = num_attention_heads
28
+ self.intermediate_size = intermediate_size
29
+ self.hidden_dropout_prob = hidden_dropout_prob
30
+ self.attention_probs_dropout_prob = attention_probs_dropout_prob
31
+ self.max_position_embeddings = max_position_embeddings
32
+ self.output_all_encoded_layers = output_all_encoded_layers
33
+ self.position_bucket_size = position_bucket_size
34
+ self.layer_norm_eps = layer_norm_eps
modeling_ltgbert.py ADDED
@@ -0,0 +1,637 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import math
2
+ from typing import List, Optional, Tuple, Union
3
+
4
+ import torch
5
+ import torch.nn as nn
6
+ import torch.nn.functional as F
7
+ from torch.utils import checkpoint
8
+
9
+ from .configuration_ltgbert import LtgbertConfig
10
+ from transformers.modeling_utils import PreTrainedModel
11
+ from transformers.activations import gelu_new
12
+ from transformers.modeling_outputs import (
13
+ MaskedLMOutput,
14
+ MultipleChoiceModelOutput,
15
+ QuestionAnsweringModelOutput,
16
+ SequenceClassifierOutput,
17
+ TokenClassifierOutput,
18
+ BaseModelOutput
19
+ )
20
+ from transformers.pytorch_utils import softmax_backward_data
21
+
22
+
23
+ class Encoder(nn.Module):
24
+ def __init__(self, config, activation_checkpointing=False):
25
+ super().__init__()
26
+ self.layers = nn.ModuleList([EncoderLayer(config) for _ in range(config.num_hidden_layers)])
27
+
28
+ for i, layer in enumerate(self.layers):
29
+ layer.mlp.mlp[1].weight.data *= math.sqrt(1.0 / (2.0 * (1 + i)))
30
+ layer.mlp.mlp[-2].weight.data *= math.sqrt(1.0 / (2.0 * (1 + i)))
31
+
32
+ self.activation_checkpointing = activation_checkpointing
33
+
34
+ def forward(self, hidden_states, attention_mask, relative_embedding):
35
+ hidden_states, attention_probs = [hidden_states], []
36
+
37
+ for layer in self.layers:
38
+ if self.activation_checkpointing:
39
+ hidden_state, attention_p = checkpoint.checkpoint(layer, hidden_states[-1], attention_mask, relative_embedding)
40
+ else:
41
+ hidden_state, attention_p = layer(hidden_states[-1], attention_mask, relative_embedding)
42
+
43
+ hidden_states.append(hidden_state)
44
+ attention_probs.append(attention_p)
45
+
46
+ return hidden_states, attention_probs
47
+
48
+
49
+ class MaskClassifier(nn.Module):
50
+ def __init__(self, config, subword_embedding):
51
+ super().__init__()
52
+ self.nonlinearity = nn.Sequential(
53
+ nn.LayerNorm(config.hidden_size, config.layer_norm_eps, elementwise_affine=False),
54
+ nn.Linear(config.hidden_size, config.hidden_size),
55
+ nn.GELU(),
56
+ nn.LayerNorm(config.hidden_size, config.layer_norm_eps, elementwise_affine=False),
57
+ nn.Dropout(config.hidden_dropout_prob),
58
+ nn.Linear(subword_embedding.size(1), subword_embedding.size(0))
59
+ )
60
+
61
+ def forward(self, x, masked_lm_labels=None):
62
+ if masked_lm_labels is not None:
63
+ x = torch.index_select(x.flatten(0, 1), 0, torch.nonzero(masked_lm_labels.flatten() != -100).squeeze())
64
+ x = self.nonlinearity(x)
65
+ return x
66
+
67
+
68
+ class EncoderLayer(nn.Module):
69
+ def __init__(self, config):
70
+ super().__init__()
71
+ self.attention = Attention(config)
72
+ self.mlp = FeedForward(config)
73
+
74
+ def forward(self, x, padding_mask, relative_embedding):
75
+ attention_output, attention_probs = self.attention(x, padding_mask, relative_embedding)
76
+ x = x + attention_output
77
+ x = x + self.mlp(x)
78
+ return x, attention_probs
79
+
80
+
81
+ class GeGLU(nn.Module):
82
+ def forward(self, x):
83
+ x, gate = x.chunk(2, dim=-1)
84
+ x = x * gelu_new(gate)
85
+ return x
86
+
87
+
88
+ class FeedForward(nn.Module):
89
+ def __init__(self, config):
90
+ super().__init__()
91
+ self.mlp = nn.Sequential(
92
+ nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps, elementwise_affine=False),
93
+ nn.Linear(config.hidden_size, 2*config.intermediate_size, bias=False),
94
+ GeGLU(),
95
+ nn.LayerNorm(config.intermediate_size, eps=config.layer_norm_eps, elementwise_affine=False),
96
+ nn.Linear(config.intermediate_size, config.hidden_size, bias=False),
97
+ nn.Dropout(config.hidden_dropout_prob)
98
+ )
99
+
100
+ def forward(self, x):
101
+ return self.mlp(x)
102
+
103
+
104
+ class MaskedSoftmax(torch.autograd.Function):
105
+ @staticmethod
106
+ def forward(self, x, mask, dim):
107
+ self.dim = dim
108
+ x.masked_fill_(mask, float('-inf'))
109
+ x = torch.softmax(x, self.dim)
110
+ x.masked_fill_(mask, 0.0)
111
+ self.save_for_backward(x)
112
+ return x
113
+
114
+ @staticmethod
115
+ def backward(self, grad_output):
116
+ output, = self.saved_tensors
117
+ input_grad = softmax_backward_data(self, grad_output, output, self.dim, output)
118
+ return input_grad, None, None
119
+
120
+
121
+ class Attention(nn.Module):
122
+ def __init__(self, config):
123
+ super().__init__()
124
+
125
+ self.config = config
126
+
127
+ if config.hidden_size % config.num_attention_heads != 0:
128
+ raise ValueError(f"The hidden size {config.hidden_size} is not a multiple of the number of attention heads {config.num_attention_heads}")
129
+
130
+ self.hidden_size = config.hidden_size
131
+ self.num_heads = config.num_attention_heads
132
+ self.head_size = config.hidden_size // config.num_attention_heads
133
+
134
+ self.in_proj_qk = nn.Linear(config.hidden_size, 2*config.hidden_size, bias=True)
135
+ self.in_proj_v = nn.Linear(config.hidden_size, config.hidden_size, bias=True)
136
+ self.out_proj = nn.Linear(config.hidden_size, config.hidden_size, bias=True)
137
+
138
+ self.pre_layer_norm = nn.LayerNorm(config.hidden_size, config.layer_norm_eps, elementwise_affine=False)
139
+ self.post_layer_norm = nn.LayerNorm(config.hidden_size, config.layer_norm_eps, elementwise_affine=True)
140
+
141
+ position_indices = torch.arange(config.max_position_embeddings, dtype=torch.long).unsqueeze(1) \
142
+ - torch.arange(config.max_position_embeddings, dtype=torch.long).unsqueeze(0)
143
+ position_indices = self.make_log_bucket_position(position_indices, config.position_bucket_size, config.max_position_embeddings)
144
+ position_indices = config.position_bucket_size - 1 + position_indices
145
+ self.register_buffer("position_indices", position_indices, persistent=True)
146
+
147
+ self.dropout = nn.Dropout(config.attention_probs_dropout_prob)
148
+ self.scale = 1.0 / math.sqrt(3 * self.head_size)
149
+
150
+ def make_log_bucket_position(self, relative_pos, bucket_size, max_position):
151
+ sign = torch.sign(relative_pos)
152
+ mid = bucket_size // 2
153
+ abs_pos = torch.where((relative_pos < mid) & (relative_pos > -mid), mid - 1, torch.abs(relative_pos).clamp(max=max_position - 1))
154
+ log_pos = torch.ceil(torch.log(abs_pos / mid) / math.log((max_position-1) / mid) * (mid - 1)).int() + mid
155
+ bucket_pos = torch.where(abs_pos <= mid, relative_pos, log_pos * sign).long()
156
+ return bucket_pos
157
+
158
+ def compute_attention_scores(self, hidden_states, relative_embedding):
159
+ key_len, batch_size, _ = hidden_states.size()
160
+ query_len = key_len
161
+
162
+ if self.position_indices.size(0) < query_len:
163
+ position_indices = torch.arange(query_len, dtype=torch.long).unsqueeze(1) \
164
+ - torch.arange(query_len, dtype=torch.long).unsqueeze(0)
165
+ position_indices = self.make_log_bucket_position(position_indices, self.config.position_bucket_size, 512)
166
+ position_indices = self.config.position_bucket_size - 1 + position_indices
167
+ self.position_indices = position_indices.to(hidden_states.device)
168
+
169
+ hidden_states = self.pre_layer_norm(hidden_states)
170
+
171
+ query, key = self.in_proj_qk(hidden_states).chunk(2, dim=2) # shape: [T, B, D]
172
+ value = self.in_proj_v(hidden_states) # shape: [T, B, D]
173
+
174
+ query = query.reshape(query_len, batch_size * self.num_heads, self.head_size).transpose(0, 1)
175
+ key = key.reshape(key_len, batch_size * self.num_heads, self.head_size).transpose(0, 1)
176
+ value = value.view(key_len, batch_size * self.num_heads, self.head_size).transpose(0, 1)
177
+
178
+ attention_scores = torch.bmm(query, key.transpose(1, 2) * self.scale)
179
+
180
+ query_pos, key_pos = self.in_proj_qk(self.dropout(relative_embedding)).chunk(2, dim=-1) # shape: [2T-1, D]
181
+ query_pos = query_pos.view(-1, self.num_heads, self.head_size) # shape: [2T-1, H, D]
182
+ key_pos = key_pos.view(-1, self.num_heads, self.head_size) # shape: [2T-1, H, D]
183
+
184
+ query = query.view(batch_size, self.num_heads, query_len, self.head_size)
185
+ key = key.view(batch_size, self.num_heads, query_len, self.head_size)
186
+
187
+ attention_c_p = torch.einsum("bhqd,khd->bhqk", query, key_pos.squeeze(1) * self.scale)
188
+ attention_p_c = torch.einsum("bhkd,qhd->bhqk", key * self.scale, query_pos.squeeze(1))
189
+
190
+ position_indices = self.position_indices[:query_len, :key_len].expand(batch_size, self.num_heads, -1, -1)
191
+ attention_c_p = attention_c_p.gather(3, position_indices)
192
+ attention_p_c = attention_p_c.gather(2, position_indices)
193
+
194
+ attention_scores = attention_scores.view(batch_size, self.num_heads, query_len, key_len)
195
+ attention_scores.add_(attention_c_p)
196
+ attention_scores.add_(attention_p_c)
197
+
198
+ return attention_scores, value
199
+
200
+ def compute_output(self, attention_probs, value):
201
+ attention_probs = self.dropout(attention_probs)
202
+ context = torch.bmm(attention_probs.flatten(0, 1), value) # shape: [B*H, Q, D]
203
+ context = context.transpose(0, 1).reshape(context.size(1), -1, self.hidden_size) # shape: [Q, B, H*D]
204
+ context = self.out_proj(context)
205
+ context = self.post_layer_norm(context)
206
+ context = self.dropout(context)
207
+ return context
208
+
209
+ def forward(self, hidden_states, attention_mask, relative_embedding):
210
+ attention_scores, value = self.compute_attention_scores(hidden_states, relative_embedding)
211
+ attention_probs = MaskedSoftmax.apply(attention_scores, attention_mask, -1)
212
+ return self.compute_output(attention_probs, value), attention_probs.detach()
213
+
214
+
215
+ class Embedding(nn.Module):
216
+ def __init__(self, config):
217
+ super().__init__()
218
+ self.hidden_size = config.hidden_size
219
+
220
+ self.word_embedding = nn.Embedding(config.vocab_size, config.hidden_size)
221
+ self.word_layer_norm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps, elementwise_affine=False)
222
+ self.dropout = nn.Dropout(config.hidden_dropout_prob)
223
+
224
+ self.relative_embedding = nn.Parameter(torch.empty(2 * config.position_bucket_size - 1, config.hidden_size))
225
+ self.relative_layer_norm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
226
+
227
+ def forward(self, input_ids):
228
+ word_embedding = self.dropout(self.word_layer_norm(self.word_embedding(input_ids)))
229
+ relative_embeddings = self.relative_layer_norm(self.relative_embedding)
230
+ return word_embedding, relative_embeddings
231
+
232
+
233
+ #
234
+ # HuggingFace wrappers
235
+ #
236
+
237
+ class LtgbertPreTrainedModel(PreTrainedModel):
238
+ config_class = LtgbertConfig
239
+ supports_gradient_checkpointing = True
240
+
241
+ def _set_gradient_checkpointing(self, module, value=False):
242
+ if isinstance(module, Encoder):
243
+ module.activation_checkpointing = value
244
+
245
+ def _init_weights(self, module):
246
+ std = math.sqrt(2.0 / (5.0 * self.hidden_size))
247
+
248
+ if isinstance(module, nn.Linear):
249
+ nn.init.trunc_normal_(module.weight.data, mean=0.0, std=std, a=-2*std, b=2*std)
250
+ if module.bias is not None:
251
+ module.bias.data.zero_()
252
+ elif isinstance(module, nn.Embedding):
253
+ nn.init.trunc_normal_(module.weight.data, mean=0.0, std=std, a=-2*std, b=2*std)
254
+ elif isinstance(module, nn.LayerNorm):
255
+ module.bias.data.zero_()
256
+ module.weight.data.fill_(1.0)
257
+
258
+
259
+ class LtgbertModel(LtgbertPreTrainedModel):
260
+ def __init__(self, config, add_mlm_layer=False, gradient_checkpointing=False, **kwargs):
261
+ super().__init__(config, **kwargs)
262
+ self.config = config
263
+ self.hidden_size = config.hidden_size
264
+
265
+ self.embedding = Embedding(config)
266
+ self.transformer = Encoder(config, activation_checkpointing=gradient_checkpointing)
267
+ self.classifier = MaskClassifier(config, self.embedding.word_embedding.weight) if add_mlm_layer else None
268
+
269
+
270
+ def get_input_embeddings(self):
271
+ return self.embedding.word_embedding
272
+
273
+ def set_input_embeddings(self, value):
274
+ self.embedding.word_embedding = value
275
+
276
+ def get_contextualized_embeddings(
277
+ self,
278
+ input_ids: Optional[torch.Tensor] = None,
279
+ attention_mask: Optional[torch.Tensor] = None
280
+ ) -> List[torch.Tensor]:
281
+ if input_ids is not None:
282
+ input_shape = input_ids.size()
283
+ else:
284
+ raise ValueError("You have to specify input_ids")
285
+
286
+ batch_size, seq_length = input_shape
287
+ device = input_ids.device
288
+
289
+ if attention_mask is None:
290
+ attention_mask = torch.zeros(batch_size, seq_length, dtype=torch.bool, device=device)
291
+ else:
292
+ attention_mask = ~attention_mask.bool()
293
+ attention_mask = attention_mask.unsqueeze(1).unsqueeze(2)
294
+
295
+ static_embeddings, relative_embedding = self.embedding(input_ids.t())
296
+ contextualized_embeddings, attention_probs = self.transformer(static_embeddings, attention_mask, relative_embedding)
297
+ contextualized_embeddings = [e.transpose(0, 1) for e in contextualized_embeddings]
298
+ last_layer = contextualized_embeddings[-1]
299
+ contextualized_embeddings = [contextualized_embeddings[0]] + [
300
+ contextualized_embeddings[i] - contextualized_embeddings[i - 1]
301
+ for i in range(1, len(contextualized_embeddings))
302
+ ]
303
+ return last_layer, contextualized_embeddings, attention_probs
304
+
305
+ def forward(
306
+ self,
307
+ input_ids: Optional[torch.Tensor] = None,
308
+ attention_mask: Optional[torch.Tensor] = None,
309
+ token_type_ids: Optional[torch.Tensor] = None,
310
+ position_ids: Optional[torch.Tensor] = None,
311
+ output_hidden_states: Optional[bool] = None,
312
+ output_attentions: Optional[bool] = None,
313
+ return_dict: Optional[bool] = None,
314
+ **kwargs
315
+ ) -> Union[Tuple[torch.Tensor], BaseModelOutput]:
316
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
317
+
318
+ sequence_output, contextualized_embeddings, attention_probs = self.get_contextualized_embeddings(input_ids, attention_mask)
319
+
320
+ if not return_dict:
321
+ return (
322
+ sequence_output,
323
+ *([contextualized_embeddings] if output_hidden_states else []),
324
+ *([attention_probs] if output_attentions else [])
325
+ )
326
+
327
+ return BaseModelOutput(
328
+ last_hidden_state=sequence_output,
329
+ hidden_states=contextualized_embeddings if output_hidden_states else None,
330
+ attentions=attention_probs if output_attentions else None
331
+ )
332
+
333
+
334
+ class LtgbertForMaskedLM(LtgbertModel):
335
+ _keys_to_ignore_on_load_unexpected = ["head"]
336
+
337
+ def __init__(self, config, **kwargs):
338
+ super().__init__(config, add_mlm_layer=True, **kwargs)
339
+
340
+ def get_output_embeddings(self):
341
+ return self.classifier.nonlinearity[-1].weight
342
+
343
+ def set_output_embeddings(self, new_embeddings):
344
+ self.classifier.nonlinearity[-1].weight = new_embeddings
345
+
346
+ def forward(
347
+ self,
348
+ input_ids: Optional[torch.Tensor] = None,
349
+ attention_mask: Optional[torch.Tensor] = None,
350
+ token_type_ids: Optional[torch.Tensor] = None,
351
+ position_ids: Optional[torch.Tensor] = None,
352
+ output_hidden_states: Optional[bool] = None,
353
+ output_attentions: Optional[bool] = None,
354
+ return_dict: Optional[bool] = None,
355
+ labels: Optional[torch.LongTensor] = None,
356
+ **kwargs
357
+ ) -> Union[Tuple[torch.Tensor], MaskedLMOutput]:
358
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
359
+
360
+ sequence_output, contextualized_embeddings, attention_probs = self.get_contextualized_embeddings(input_ids, attention_mask)
361
+ subword_prediction = self.classifier(sequence_output)
362
+ subword_prediction[:, :, :106+1] = float("-inf")
363
+
364
+ masked_lm_loss = None
365
+ if labels is not None:
366
+ masked_lm_loss = F.cross_entropy(subword_prediction.flatten(0, 1), labels.flatten())
367
+
368
+ if not return_dict:
369
+ output = (
370
+ subword_prediction,
371
+ *([contextualized_embeddings] if output_hidden_states else []),
372
+ *([attention_probs] if output_attentions else [])
373
+ )
374
+ return ((masked_lm_loss,) + output) if masked_lm_loss is not None else output
375
+
376
+ return MaskedLMOutput(
377
+ loss=masked_lm_loss,
378
+ logits=subword_prediction,
379
+ hidden_states=contextualized_embeddings if output_hidden_states else None,
380
+ attentions=attention_probs if output_attentions else None
381
+ )
382
+
383
+
384
+ class Classifier(nn.Module):
385
+ def __init__(self, config, num_labels: int):
386
+ super().__init__()
387
+
388
+ drop_out = getattr(config, "cls_dropout", None)
389
+ drop_out = config.hidden_dropout_prob if drop_out is None else drop_out
390
+
391
+ self.nonlinearity = nn.Sequential(
392
+ nn.LayerNorm(config.hidden_size, config.layer_norm_eps, elementwise_affine=False),
393
+ nn.Linear(config.hidden_size, config.hidden_size),
394
+ nn.GELU(),
395
+ nn.LayerNorm(config.hidden_size, config.layer_norm_eps, elementwise_affine=False),
396
+ nn.Dropout(drop_out),
397
+ nn.Linear(config.hidden_size, num_labels)
398
+ )
399
+
400
+ def forward(self, x):
401
+ x = self.nonlinearity(x)
402
+ return x
403
+
404
+
405
+ class LtgbertForSequenceClassification(LtgbertModel):
406
+ _keys_to_ignore_on_load_unexpected = ["classifier"]
407
+ _keys_to_ignore_on_load_missing = ["head"]
408
+
409
+ def __init__(self, config, **kwargs):
410
+ super().__init__(config, add_mlm_layer=False, **kwargs)
411
+
412
+ self.num_labels = config.num_labels
413
+ self.head = Classifier(config, self.num_labels)
414
+
415
+ def forward(
416
+ self,
417
+ input_ids: Optional[torch.Tensor] = None,
418
+ attention_mask: Optional[torch.Tensor] = None,
419
+ token_type_ids: Optional[torch.Tensor] = None,
420
+ position_ids: Optional[torch.Tensor] = None,
421
+ output_attentions: Optional[bool] = None,
422
+ output_hidden_states: Optional[bool] = None,
423
+ return_dict: Optional[bool] = None,
424
+ labels: Optional[torch.LongTensor] = None,
425
+ **kwargs
426
+ ) -> Union[Tuple[torch.Tensor], SequenceClassifierOutput]:
427
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
428
+
429
+ sequence_output, contextualized_embeddings, attention_probs = self.get_contextualized_embeddings(input_ids, attention_mask)
430
+ logits = self.head(sequence_output[:, 0, :])
431
+
432
+ loss = None
433
+ if labels is not None:
434
+ if self.config.problem_type is None:
435
+ if self.num_labels == 1:
436
+ self.config.problem_type = "regression"
437
+ elif self.num_labels > 1 and (labels.dtype == torch.long or labels.dtype == torch.int):
438
+ self.config.problem_type = "single_label_classification"
439
+ else:
440
+ self.config.problem_type = "multi_label_classification"
441
+
442
+ if self.config.problem_type == "regression":
443
+ loss_fct = nn.MSELoss()
444
+ if self.num_labels == 1:
445
+ loss = loss_fct(logits.squeeze(), labels.squeeze())
446
+ else:
447
+ loss = loss_fct(logits, labels)
448
+ elif self.config.problem_type == "single_label_classification":
449
+ loss_fct = nn.CrossEntropyLoss()
450
+ loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1))
451
+ elif self.config.problem_type == "multi_label_classification":
452
+ loss_fct = nn.BCEWithLogitsLoss()
453
+ loss = loss_fct(logits, labels)
454
+
455
+ if not return_dict:
456
+ output = (
457
+ logits,
458
+ *([contextualized_embeddings] if output_hidden_states else []),
459
+ *([attention_probs] if output_attentions else [])
460
+ )
461
+ return ((loss,) + output) if loss is not None else output
462
+
463
+ return SequenceClassifierOutput(
464
+ loss=loss,
465
+ logits=logits,
466
+ hidden_states=contextualized_embeddings if output_hidden_states else None,
467
+ attentions=attention_probs if output_attentions else None
468
+ )
469
+
470
+
471
+ class LtgbertForTokenClassification(LtgbertModel):
472
+ _keys_to_ignore_on_load_unexpected = ["classifier"]
473
+ _keys_to_ignore_on_load_missing = ["head"]
474
+
475
+ def __init__(self, config, **kwargs):
476
+ super().__init__(config, add_mlm_layer=False, **kwargs)
477
+
478
+ self.num_labels = config.num_labels
479
+ self.head = Classifier(config, self.num_labels)
480
+
481
+ def forward(
482
+ self,
483
+ input_ids: Optional[torch.Tensor] = None,
484
+ attention_mask: Optional[torch.Tensor] = None,
485
+ token_type_ids: Optional[torch.Tensor] = None,
486
+ position_ids: Optional[torch.Tensor] = None,
487
+ output_attentions: Optional[bool] = None,
488
+ output_hidden_states: Optional[bool] = None,
489
+ return_dict: Optional[bool] = None,
490
+ labels: Optional[torch.LongTensor] = None,
491
+ **kwargs
492
+ ) -> Union[Tuple[torch.Tensor], TokenClassifierOutput]:
493
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
494
+
495
+ sequence_output, contextualized_embeddings, attention_probs = self.get_contextualized_embeddings(input_ids, attention_mask)
496
+ logits = self.head(sequence_output)
497
+
498
+ loss = None
499
+ if labels is not None:
500
+ loss_fct = nn.CrossEntropyLoss()
501
+ loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1))
502
+
503
+ if not return_dict:
504
+ output = (
505
+ logits,
506
+ *([contextualized_embeddings] if output_hidden_states else []),
507
+ *([attention_probs] if output_attentions else [])
508
+ )
509
+ return ((loss,) + output) if loss is not None else output
510
+
511
+ return TokenClassifierOutput(
512
+ loss=loss,
513
+ logits=logits,
514
+ hidden_states=contextualized_embeddings if output_hidden_states else None,
515
+ attentions=attention_probs if output_attentions else None
516
+ )
517
+
518
+
519
+ class LtgbertForQuestionAnswering(LtgbertModel):
520
+ _keys_to_ignore_on_load_unexpected = ["classifier"]
521
+ _keys_to_ignore_on_load_missing = ["head"]
522
+
523
+ def __init__(self, config, **kwargs):
524
+ super().__init__(config, add_mlm_layer=False, **kwargs)
525
+
526
+ self.num_labels = config.num_labels
527
+ self.head = Classifier(config, self.num_labels)
528
+
529
+ def forward(
530
+ self,
531
+ input_ids: Optional[torch.Tensor] = None,
532
+ attention_mask: Optional[torch.Tensor] = None,
533
+ token_type_ids: Optional[torch.Tensor] = None,
534
+ position_ids: Optional[torch.Tensor] = None,
535
+ output_attentions: Optional[bool] = None,
536
+ output_hidden_states: Optional[bool] = None,
537
+ return_dict: Optional[bool] = None,
538
+ start_positions: Optional[torch.Tensor] = None,
539
+ end_positions: Optional[torch.Tensor] = None,
540
+ **kwargs
541
+ ) -> Union[Tuple[torch.Tensor], QuestionAnsweringModelOutput]:
542
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
543
+
544
+ sequence_output, contextualized_embeddings, attention_probs = self.get_contextualized_embeddings(input_ids, attention_mask)
545
+ logits = self.head(sequence_output)
546
+
547
+ start_logits, end_logits = logits.split(1, dim=-1)
548
+ start_logits = start_logits.squeeze(-1).contiguous()
549
+ end_logits = end_logits.squeeze(-1).contiguous()
550
+
551
+ total_loss = None
552
+ if start_positions is not None and end_positions is not None:
553
+ # If we are on multi-GPU, split add a dimension
554
+ if len(start_positions.size()) > 1:
555
+ start_positions = start_positions.squeeze(-1)
556
+ if len(end_positions.size()) > 1:
557
+ end_positions = end_positions.squeeze(-1)
558
+
559
+ # sometimes the start/end positions are outside our model inputs, we ignore these terms
560
+ ignored_index = start_logits.size(1)
561
+ start_positions = start_positions.clamp(0, ignored_index)
562
+ end_positions = end_positions.clamp(0, ignored_index)
563
+
564
+ loss_fct = nn.CrossEntropyLoss(ignore_index=ignored_index)
565
+ start_loss = loss_fct(start_logits, start_positions)
566
+ end_loss = loss_fct(end_logits, end_positions)
567
+ total_loss = (start_loss + end_loss) / 2
568
+
569
+ if not return_dict:
570
+ output = (
571
+ start_logits,
572
+ end_logits,
573
+ *([contextualized_embeddings] if output_hidden_states else []),
574
+ *([attention_probs] if output_attentions else [])
575
+ )
576
+ return ((total_loss,) + output) if total_loss is not None else output
577
+
578
+ return QuestionAnsweringModelOutput(
579
+ loss=total_loss,
580
+ start_logits=start_logits,
581
+ end_logits=end_logits,
582
+ hidden_states=contextualized_embeddings if output_hidden_states else None,
583
+ attentions=attention_probs if output_attentions else None
584
+ )
585
+
586
+
587
+ class LtgbertForMultipleChoice(LtgbertModel):
588
+ _keys_to_ignore_on_load_unexpected = ["classifier"]
589
+ _keys_to_ignore_on_load_missing = ["head"]
590
+
591
+ def __init__(self, config, **kwargs):
592
+ super().__init__(config, add_mlm_layer=False, **kwargs)
593
+
594
+ self.num_labels = getattr(config, "num_labels", 2)
595
+ self.head = Classifier(config, self.num_labels)
596
+
597
+ def forward(
598
+ self,
599
+ input_ids: Optional[torch.Tensor] = None,
600
+ attention_mask: Optional[torch.Tensor] = None,
601
+ token_type_ids: Optional[torch.Tensor] = None,
602
+ position_ids: Optional[torch.Tensor] = None,
603
+ labels: Optional[torch.Tensor] = None,
604
+ output_attentions: Optional[bool] = None,
605
+ output_hidden_states: Optional[bool] = None,
606
+ return_dict: Optional[bool] = None,
607
+ **kwargs
608
+ ) -> Union[Tuple[torch.Tensor], MultipleChoiceModelOutput]:
609
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
610
+ num_choices = input_ids.shape[1]
611
+
612
+ flat_input_ids = input_ids.view(-1, input_ids.size(-1))
613
+ flat_attention_mask = attention_mask.view(-1, attention_mask.size(-1)) if attention_mask is not None else None
614
+
615
+ sequence_output, contextualized_embeddings, attention_probs = self.get_contextualized_embeddings(flat_input_ids, flat_attention_mask)
616
+ logits = self.head(sequence_output)
617
+ reshaped_logits = logits.view(-1, num_choices)
618
+
619
+ loss = None
620
+ if labels is not None:
621
+ loss_fct = nn.CrossEntropyLoss()
622
+ loss = loss_fct(reshaped_logits, labels)
623
+
624
+ if not return_dict:
625
+ output = (
626
+ reshaped_logits,
627
+ *([contextualized_embeddings] if output_hidden_states else []),
628
+ *([attention_probs] if output_attentions else [])
629
+ )
630
+ return ((loss,) + output) if loss is not None else output
631
+
632
+ return MultipleChoiceModelOutput(
633
+ loss=loss,
634
+ logits=reshaped_logits,
635
+ hidden_states=contextualized_embeddings if output_hidden_states else None,
636
+ attentions=attention_probs if output_attentions else None
637
+ )
pytorch_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:51f6a2e7aed7b0e0e991d876dc48bb267350dc3a914b9093ad758419b2bc0252
3
+ size 525164345
spacial_tokens_map.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"bos_token": "[BOS]", "eos_token": "[EOS]", "unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]"}
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "tokenizer_class": "PreTrainedTokenizerFast",
3
+ "bos_token": "[BOS]",
4
+ "eos_token": "[EOS]",
5
+ "unk_token": "[UNK]",
6
+ "sep_token": "[SEP]",
7
+ "pad_token": "[PAD]",
8
+ "cls_token": "[CLS]",
9
+ "mask_token": "[MASK]"
10
+ }