Upload BERT_Arch

config.json

@@ -2,6 +2,10 @@
   "architectures": [
     "BERT_Arch"
   ],
+  "auto_map": {
+    "AutoConfig": "model_config.PragFormerConfig",
+    "AutoModel": "model.BERT_Arch"
+  },
   "bert": {
     "_commit_hash": "43cf2d48e8c75d255dccab2a19e40d4774fd8853",
     "_name_or_path": "NTUYG/DeepSCC-RoBERTa",

model.py

@@ -0,0 +1,48 @@
+from transformers import AutoModel, AutoConfig
+import torch.nn as nn
+from transformers import BertPreTrainedModel, AutoModel, PreTrainedModel
+from model_config import PragFormerConfig
+
+
+
+class BERT_Arch(PreTrainedModel): #(BertPreTrainedModel):
+    config_class = PragFormerConfig
+
+    def __init__(self, config):
+        super().__init__(config)
+        print(config.bert)
+        self.bert = AutoModel.from_pretrained(config.bert['_name_or_path'])
+
+        # dropout layer
+        self.dropout = nn.Dropout(config.dropout)
+
+        # relu activation function
+        self.relu = nn.ReLU()
+
+        # dense layer 1
+        self.fc1 = nn.Linear(self.config.bert['hidden_size'], config.fc1)
+        # self.fc1 = nn.Linear(768, 512)
+
+        # dense layer 2 (Output layer)
+        self.fc2 = nn.Linear(config.fc1, config.fc2)
+
+        # softmax activation function
+        self.softmax = nn.LogSoftmax(dim = config.softmax_dim)
+
+    # define the forward pass
+    def forward(self, input_ids, attention_mask):
+        # pass the inputs to the model
+        _, cls_hs = self.bert(input_ids, attention_mask = attention_mask, return_dict=False)
+
+        x = self.fc1(cls_hs)
+
+        x = self.relu(x)
+
+        x = self.dropout(x)
+        
+        # output layer
+        x = self.fc2(x)
+
+        # apply softmax activation
+        x = self.softmax(x)
+        return x

model_config.py

@@ -0,0 +1,13 @@
+from transformers import PretrainedConfig
+
+
+class PragFormerConfig(PretrainedConfig):
+    model_type = "pragformer"
+
+    def __init__(self, bert=None, dropout=0.2, fc1=512, fc2=2, softmax_dim=1, **kwargs):
+        self.bert = bert
+        self.dropout = dropout
+        self.fc1 = fc1
+        self.fc2 = fc2
+        self.softmax_dim = softmax_dim
+        super().__init__(**kwargs)

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:816862e8a2fc62fac62de92357c9394c6649e27b134f3da267e4dbfedad0e544
+oid sha256:c662ec6193215ce2edf8b419eea4c8d8e8e907282811a58565093d0797b6638f
 size 500230377