Upload model

config.json

@@ -0,0 +1,16 @@
+{
+  "architectures": [
+    "OffensivenessEstimationModel"
+  ],
+  "auto_map": {
+    "AutoConfig": "configuration.OffensivenessEstimationConfig",
+    "AutoModel": "modeling.OffensivenessEstimationModel"
+  },
+  "dropout_rate": 0.1,
+  "language_model": "studio-ousia/luke-japanese-base-lite",
+  "model_type": "offensiveness_estimation",
+  "output_class_num": 11,
+  "reinit_n_layers": 1,
+  "torch_dtype": "float32",
+  "transformers_version": "4.35.2"
+}

configuration.py

@@ -0,0 +1,24 @@
+from transformers import PretrainedConfig
+from typing import List
+
+class OffensivenessEstimationConfig(PretrainedConfig):
+    model_type = "offensiveness_estimation"
+
+    def __init__(
+        self,
+        language_model: str   = 'studio-ousia/luke-japanese-base-lite',
+        output_class_num: int = 11,
+        reinit_n_layers: int  = 1,
+        dropout_rate: float   = 0.1,
+        **kwargs,
+    ):
+        # if block_type not in ["basic", "bottleneck"]:
+        #     raise ValueError(f"`block_type` must be 'basic' or bottleneck', got {block_type}.")
+        # if stem_type not in ["", "deep", "deep-tiered"]:
+        #     raise ValueError(f"`stem_type` must be '', 'deep' or 'deep-tiered', got {stem_type}.")
+
+        self.language_model   = language_model
+        self.output_class_num = output_class_num
+        self.reinit_n_layers  = reinit_n_layers
+        self.dropout_rate     = dropout_rate
+        super().__init__(**kwargs)

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:63c3389d4b0f0650fc41ec64abf8fcecc9b70a7deaad8cf5215dbbdbc983b25a
+size 532341340

modeling.py

@@ -0,0 +1,52 @@
+from transformers import PreTrainedModel
+from .configuration import *
+
+import torch.nn as nn
+import torch
+from transformers import AutoModel
+
+class OffensivenessEstimationModel(PreTrainedModel):
+    config_class = OffensivenessEstimationConfig
+
+    def __init__(self, config):
+        super().__init__(config)
+        self.text_encoder = PretrainedLanguageModel(config)
+        self.decoder = nn.Sequential(
+            nn.Dropout(p=config.dropout_rate),
+            nn.Linear(768, config.output_class_num)
+        )
+
+    def forward(self, ids, mask):
+        h = self.text_encoder(ids, mask)
+        output = self.decoder(h)
+        return output
+
+class PretrainedLanguageModel(PreTrainedModel):
+    config_class = OffensivenessEstimationConfig
+
+    def __init__(self, config):
+        super().__init__(config)
+        self.language_model = AutoModel.from_pretrained(config.language_model)
+        self.reinit_n_layers = config.reinit_n_layers
+        if self.reinit_n_layers > 0:
+            self._do_reinit()
+
+    def _do_reinit(self):
+        # Re-init last n layers.
+        for layer in self.language_model.encoder.layer[-1*self.reinit_n_layers:]:
+            for module in layer.modules():
+                if isinstance(module, nn.Linear):
+                    module.weight.data.normal_(mean=0.0, std=self.language_model.config.initializer_range)
+                    if module.bias is not None:
+                        module.bias.data.zero_()
+                elif isinstance(module, nn.Embedding):
+                    module.weight.data.normal_(mean=0.0, std=self.language_model.config.initializer_range)
+                    if module.padding_idx is not None:
+                        module.weight.data[module.padding_idx].zero_()
+                elif isinstance(module, nn.LayerNorm):
+                    module.bias.data.zero_()
+                    module.weight.data.fill_(1.0)
+
+    def forward(self, ids, mask):
+        output = self.language_model(ids, attention_mask=mask)
+        return output[0][:,0,:]