Add HAT implementation files

modelling_hat.py

@@ -1839,8 +1839,6 @@ class HATForSequenceClassification(HATPreTrainedModel):
             config.classifier_dropout if config.classifier_dropout is not None else config.hidden_dropout_prob
         )
         self.dropout = nn.Dropout(classifier_dropout)
-        if self.pooling != 'cls':
-            self.sentencizer = HATSentencizer(config)
         self.pooler = HATPooler(config, pooling=pooling)
         self.classifier = nn.Linear(config.hidden_size, config.num_labels)
 
@@ -1885,13 +1883,12 @@ class HATForSequenceClassification(HATPreTrainedModel):
             return_dict=return_dict,
         )
         sequence_output = outputs[0]
-        if self.pooling not in ['first', 'last']:
-            sentence_outputs = self.sentencizer(sequence_output)
-            pooled_output = self.pooler(sentence_outputs)
-        elif self.pooling == 'first':
+        if self.pooling == 'first':
             pooled_output = self.pooler(torch.unsqueeze(sequence_output[:, 0, :], 1))
         elif self.pooling == 'last':
             pooled_output = self.pooler(torch.unsqueeze(sequence_output[:, -128, :], 1))
+        else:
+            pooled_output = self.pooler(sequence_output[:, ::self.max_sentence_length])
 
         pooled_output = self.dropout(pooled_output)
         logits = self.classifier(pooled_output)
@@ -2051,8 +2048,6 @@ class HATForMultipleChoice(HATPreTrainedModel):
             config.classifier_dropout if config.classifier_dropout is not None else config.hidden_dropout_prob
         )
         self.dropout = nn.Dropout(classifier_dropout)
-        if self.pooling not in ['first', 'last']:
-            self.sentencizer = HATSentencizer(config)
         self.pooler = HATPooler(config, pooling=pooling)
         self.classifier = nn.Linear(config.hidden_size, 1)
 
@@ -2113,7 +2108,7 @@ class HATForMultipleChoice(HATPreTrainedModel):
         elif self.pooling == 'last':
             pooled_output = self.pooler(torch.unsqueeze(sequence_output[:, -128, :], 1))
         else:
-            pooled_output = self.pooler(self.sentencizer(sequence_output))
+            pooled_output = self.pooler(sequence_output[:, ::self.max_sentence_length])
 
         pooled_output = self.dropout(pooled_output)
         logits = self.classifier(pooled_output)