Uploaded config and code files

config.json

@@ -0,0 +1,26 @@
+{
+  "architectures": [
+    "GAN"
+  ],
+  "auto_map": {
+    "AutoConfig": "configuration_ganbert.GanBertConfig",
+    "AutoModelForSequenceClassification": "ganbert.GAN"
+  },
+  "batch_size": 64,
+  "device": "cuda",
+  "epochs": 10,
+  "epsilon": 1e-08,
+  "learning_rate_discriminator": 1e-05,
+  "learning_rate_generator": 1e-05,
+  "model_number": -2,
+  "model_type": "ganbert",
+  "noise_size": 100,
+  "num_hidden_layers_d": 1,
+  "num_hidden_layers_g": 2,
+  "num_train_examples": 77450,
+  "out_dropout_rate": 0.4,
+  "pos_class_weight": 10,
+  "torch_dtype": "float32",
+  "transformers_version": "4.20.1",
+  "warmup_proportion": 0.1
+}

configuration_ganbert.py

@@ -0,0 +1,73 @@
+from transformers import PretrainedConfig
+import logging
+import datasets
+from datasets import load_dataset
+import pandas as pd
+import matplotlib.pyplot as plt
+import seaborn as sns
+from datasets import load_metric
+import transformers
+import torch
+import io
+import torch.nn.functional as F
+import random
+import numpy as np
+import time
+import math
+import datetime
+import torch.nn as nn
+from torch.utils.data import Dataset,TensorDataset, DataLoader, RandomSampler, SequentialSampler
+from transformers import (
+    AutoModel,
+    AutoConfig,
+    AutoModelForSequenceClassification,
+    AutoTokenizer,
+    DataCollatorWithPadding,
+    default_data_collator,
+    set_seed,
+    get_constant_schedule_with_warmup,
+    Trainer,TrainingArguments,EarlyStoppingCallback)
+from datasets import Dataset
+import torch.nn as nn
+import torch.nn.functional as F
+import sys
+
+class GanBertConfig(PretrainedConfig):
+    model_type = "ganbert"
+
+    def __init__(
+        self,
+        out_dropout_rate = 0.4,
+        num_hidden_layers_g = 2,
+        num_hidden_layers_d = 1,
+        pos_class_weight = 10,
+        batch_size = 64,
+        noise_size = 100,
+        num_train_examples = 77450,
+        epochs = 10,
+        epsilon = 1e-08,
+        learning_rate_discriminator = 1e-05,
+        learning_rate_generator = 1e-05,
+        warmup_proportion= 0.1,      
+        model_number = -2,
+        device ='cuda',
+        **kwargs,
+    ):
+        self.out_dropout_rate=out_dropout_rate
+        self.num_hidden_layers_g=num_hidden_layers_g
+        self.num_hidden_layers_d=num_hidden_layers_d
+        self.pos_class_weight=pos_class_weight
+        self.model_number = model_number
+        self.learning_rate_discriminator=learning_rate_discriminator
+        self.learning_rate_generator=learning_rate_generator
+        self.warmup_proportion=warmup_proportion
+        self.epsilon=epsilon
+        self.num_train_examples=num_train_examples
+        self.epochs = epochs
+        self.batch_size=batch_size
+        self.noise_size = noise_size
+        if torch.cuda.is_available():
+          self.device = 'cuda'
+        else:
+          self.device = 'cpu'
+        super().__init__(**kwargs)

gan.py

@@ -0,0 +1,76 @@
+import logging
+import datasets
+from datasets import load_dataset
+import pandas as pd
+import matplotlib.pyplot as plt
+import seaborn as sns
+from datasets import load_metric
+import transformers
+import torch
+import io
+import torch.nn.functional as F
+import random
+import numpy as np
+import time
+import math
+import datetime
+import torch.nn as nn
+from torch.utils.data import Dataset,TensorDataset, DataLoader, RandomSampler, SequentialSampler
+from transformers import (
+    AutoModel,
+    AutoConfig,
+    AutoModelForSequenceClassification,
+    AutoTokenizer,
+    DataCollatorWithPadding,
+    default_data_collator,
+    set_seed,
+    get_constant_schedule_with_warmup,
+    Trainer,TrainingArguments,EarlyStoppingCallback)
+from datasets import Dataset
+import torch.nn as nn
+import torch.nn.functional as F
+import sys
+#------------------------------
+#   The Generator as in 
+#   https://www.aclweb.org/anthology/2020.acl-main.191/
+#   https://github.com/crux82/ganbert
+#------------------------------
+class Generator(nn.Module):
+    def __init__(self, noise_size=100, output_size=512, hidden_sizes=[512], dropout_rate=0.1):
+        super(Generator, self).__init__()
+        layers = []
+        hidden_sizes = [noise_size] + hidden_sizes
+        for i in range(len(hidden_sizes)-1):
+            layers.extend([nn.Linear(hidden_sizes[i], hidden_sizes[i+1]), nn.LeakyReLU(0.2, inplace=True), nn.Dropout(dropout_rate)])
+
+        layers.append(nn.Linear(hidden_sizes[-1],output_size))
+        self.layers = nn.Sequential(*layers)
+
+    def forward(self, noise):
+        output_rep = self.layers(noise)
+        return output_rep
+
+#------------------------------
+#   The Discriminator
+#   https://www.aclweb.org/anthology/2020.acl-main.191/
+#   https://github.com/crux82/ganbert
+#------------------------------
+class Discriminator(nn.Module):
+    def __init__(self, input_size=512, hidden_sizes=[512], num_labels=2, dropout_rate=0.3):
+        super(Discriminator, self).__init__()
+        self.input_dropout = nn.Dropout(p=dropout_rate)
+        layers = []
+        hidden_sizes = [input_size] + hidden_sizes
+        for i in range(len(hidden_sizes)-1):
+            layers.extend([nn.Linear(hidden_sizes[i], hidden_sizes[i+1]), nn.LeakyReLU(0.2, inplace=True), nn.Dropout(dropout_rate)])
+
+        self.layers = nn.Sequential(*layers) #per il flatten
+        self.logit = nn.Linear(hidden_sizes[-1],num_labels+1) # +1 for the probability of this sample being fake/real.
+        self.softmax = nn.Softmax(dim=-1)
+
+    def forward(self, input_rep):
+        input_rep = self.input_dropout(input_rep)
+        last_rep = self.layers(input_rep)
+        logits = self.logit(last_rep)
+        probs = self.softmax(logits)
+        return last_rep, logits, probs

ganbert.py

@@ -0,0 +1,94 @@
+from transformers import PreTrainedModel
+from .configuration_ganbert import GanBertConfig
+from .gan import Generator,Discriminator
+from transformers import PretrainedConfig
+import logging
+import datasets
+from datasets import load_dataset
+import pandas as pd
+import matplotlib.pyplot as plt
+import seaborn as sns
+from datasets import load_metric
+import transformers
+import torch
+import io
+import torch.nn.functional as F
+import random
+import numpy as np
+import time
+import math
+import datetime
+import torch.nn as nn
+from torch.utils.data import Dataset,TensorDataset, DataLoader, RandomSampler, SequentialSampler
+from transformers import (
+    AutoModel,
+    AutoConfig,
+    AutoModelForSequenceClassification,
+    AutoTokenizer,
+    DataCollatorWithPadding,
+    default_data_collator,
+    set_seed,
+    get_constant_schedule_with_warmup,
+    Trainer,TrainingArguments,EarlyStoppingCallback)
+from datasets import Dataset
+import torch.nn as nn
+import torch.nn.functional as F
+import sys
+
+class GAN(PreTrainedModel):
+  config_class = GanBertConfig
+  all_checkpoints=['bert-base-multilingual-cased',
+                 'sagorsarker/bangla-bert-base',
+                 'neuralspace-reverie/indic-transformers-bn-bert',
+                 'neuralspace-reverie/indic-transformers-bn-roberta',
+                 'distilbert-base-multilingual-cased',
+                 'neuralspace-reverie/indic-transformers-bn-distilbert',
+                 'monsoon-nlp/bangla-electra',
+                 'csebuetnlp/banglabert',
+                 'neuralspace-reverie/indic-transformers-bn-xlmroberta'
+                 ]
+  def __init__(
+      self,
+      config
+  ):
+      super().__init__(config)
+
+      self.model_name = self.all_checkpoints[config.model_number]
+      self.parent_config = AutoConfig.from_pretrained(self.model_name)
+      self.hidden_size = int(self.parent_config.hidden_size)
+      self.ns = config.noise_size
+      self.dv = config.device
+      # Define the number and width of hidden layers
+      self.hidden_levels_g = [self.hidden_size for i in range(0, config.num_hidden_layers_g)]
+      self.hidden_levels_d = [self.hidden_size for i in range(0, config.num_hidden_layers_d)]
+      self.label_list =  [0,1,2]
+      self.class_weight = torch.tensor([10,config.pos_class_weight,5],device=config.device)
+      #-------------------------------------------------
+      #   Instantiate the Generator and Discriminator
+      #-------------------------------------------------
+      self.generator = Generator(noise_size=config.noise_size, output_size=self.hidden_size, hidden_sizes=self.hidden_levels_g, dropout_rate=config.out_dropout_rate)
+      self.discriminator = Discriminator(input_size=self.hidden_size, hidden_sizes=self.hidden_levels_d,num_labels=len(self.label_list), dropout_rate=config.out_dropout_rate)
+      # Put everything in the GPU if available
+      # print(self.generator,self.discriminator)
+      self.transformer = AutoModel.from_pretrained(self.model_name,output_attentions=True)
+      
+      if config.device == 'cuda':    
+        self.generator.cuda()
+        self.discriminator.cuda()
+        self.transformer.cuda()
+  def forward(self,b_input_ids,b_input_mask):
+    # Encode real data in the Transformer
+    real_batch_size = b_input_ids.shape[0]
+    model_outputs = self.transformer(b_input_ids, attention_mask=b_input_mask)
+    # print('got transformer output')
+    
+    hidden_states = torch.mean(model_outputs[0],dim=1)
+    noise = torch.zeros(real_batch_size, self.ns, device=self.dv).uniform_(0, 1).to(self.dv)
+    gen_rep = self.generator(noise)
+    disciminator_input = torch.cat([hidden_states, gen_rep], dim=0)
+    features, logits, probs = self.discriminator(disciminator_input)
+    return model_outputs[0]
+
+if __name__ == '__main__':
+  ganconfig = GanBertConfig()
+  clickbaitmodel = GAN(ganconfig)