Upload RobertaForMaskedLMV2

AbLang_roberta_model.py

@@ -0,0 +1,41 @@
+from transformers.models.roberta.modeling_roberta import RobertaEmbeddings, RobertaModel, RobertaForMaskedLM
+from typing import Optional
+import torch
+
+class RobertaEmbeddingsV2(RobertaEmbeddings):
+    def __init__(self, config):
+        super().__init__(config)
+        self.pad_token_id = config.pad_token_id
+        self.position_embeddings = torch.nn.Embedding(config.max_position_embeddings, config.hidden_size, padding_idx=0) # here padding_idx is always 0
+
+    def forward(
+        self,
+        input_ids: torch.LongTensor,
+        token_type_ids: Optional[torch.LongTensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        past_key_values_length: int = 0,
+    ) -> torch.Tensor:
+        inputs_embeds = self.word_embeddings(input_ids)
+        position_ids = self.create_position_ids_from_input_ids(input_ids)  
+        position_embeddings = self.position_embeddings(position_ids)
+        embeddings = inputs_embeds + position_embeddings
+        return self.dropout(self.LayerNorm(embeddings))
+    
+    def create_position_ids_from_input_ids(self, input_ids: torch.LongTensor) -> torch.Tensor:
+        mask = input_ids.ne(self.pad_token_id).int()
+        return torch.cumsum(mask, dim=1).long() * mask
+    
+
+class RobertaModelV2(RobertaModel):
+    def __init__(self, config, add_pooling_layer=False):
+        super().__init__(config, add_pooling_layer=add_pooling_layer)
+        self.embeddings = RobertaEmbeddingsV2(config)
+
+
+class RobertaForMaskedLMV2(RobertaForMaskedLM):
+    def __init__(self, config):
+        super().__init__(config)
+        self.roberta = RobertaModelV2(config, add_pooling_layer=False)
+
+

config.json

@@ -1,13 +1,15 @@
 {
   "add_pooling_layer": false,
   "architectures": [
-    "BertModelV2"
+    "RobertaForMaskedLMV2"
   ],
   "attention_probs_dropout_prob": 0.1,
   "auto_map": {
-    "AutoModel": "AbLang_bert_model.BertModelV2"
+    "AutoModelForMaskedLM": "AbLang_roberta_model.RobertaForMaskedLMV2"
   },
+  "bos_token_id": 0,
   "classifier_dropout": null,
+  "eos_token_id": 2,
   "hidden_act": "gelu",
   "hidden_dropout_prob": 0.1,
   "hidden_size": 768,
@@ -15,11 +17,12 @@
   "intermediate_size": 3072,
   "layer_norm_eps": 1e-12,
   "max_position_embeddings": 160,
-  "model_type": "bert",
+  "model_type": "roberta",
   "num_attention_heads": 12,
   "num_hidden_layers": 12,
   "pad_token_id": 21,
   "position_embedding_type": "absolute",
+  "tie_word_embeddings": false,
   "torch_dtype": "float32",
   "transformers_version": "4.28.1",
   "type_vocab_size": 2,

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c9d5458446b8f723995df81e9b24b7a4635285fcb33d0d787a7e308bb16c75ea
-size 343223341
+oid sha256:508c7ea07c28cf327ae680d5b7d1ce72def49c8099991bbbe40997a772055dd7
+size 343306045