|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
import unittest |
|
|
|
from transformers import is_torch_available |
|
from transformers.models.auto import get_values |
|
from transformers.testing_utils import require_torch, slow, torch_device |
|
|
|
from .test_configuration_common import ConfigTester |
|
from .test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask |
|
|
|
|
|
if is_torch_available(): |
|
import torch |
|
|
|
from transformers import ( |
|
MODEL_FOR_PRETRAINING_MAPPING, |
|
AlbertConfig, |
|
AlbertForMaskedLM, |
|
AlbertForMultipleChoice, |
|
AlbertForPreTraining, |
|
AlbertForQuestionAnswering, |
|
AlbertForSequenceClassification, |
|
AlbertForTokenClassification, |
|
AlbertModel, |
|
) |
|
from transformers.models.albert.modeling_albert import ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST |
|
|
|
|
|
class AlbertModelTester: |
|
def __init__( |
|
self, |
|
parent, |
|
): |
|
self.parent = parent |
|
self.batch_size = 13 |
|
self.seq_length = 7 |
|
self.is_training = True |
|
self.use_input_mask = True |
|
self.use_token_type_ids = True |
|
self.use_labels = True |
|
self.vocab_size = 99 |
|
self.embedding_size = 16 |
|
self.hidden_size = 36 |
|
self.num_hidden_layers = 6 |
|
self.num_hidden_groups = 6 |
|
self.num_attention_heads = 6 |
|
self.intermediate_size = 37 |
|
self.hidden_act = "gelu" |
|
self.hidden_dropout_prob = 0.1 |
|
self.attention_probs_dropout_prob = 0.1 |
|
self.max_position_embeddings = 512 |
|
self.type_vocab_size = 16 |
|
self.type_sequence_label_size = 2 |
|
self.initializer_range = 0.02 |
|
self.num_labels = 3 |
|
self.num_choices = 4 |
|
self.scope = None |
|
|
|
def prepare_config_and_inputs(self): |
|
input_ids = ids_tensor([self.batch_size, self.seq_length], self.vocab_size) |
|
|
|
input_mask = None |
|
if self.use_input_mask: |
|
input_mask = random_attention_mask([self.batch_size, self.seq_length]) |
|
|
|
token_type_ids = None |
|
if self.use_token_type_ids: |
|
token_type_ids = ids_tensor([self.batch_size, self.seq_length], self.type_vocab_size) |
|
|
|
sequence_labels = None |
|
token_labels = None |
|
choice_labels = None |
|
if self.use_labels: |
|
sequence_labels = ids_tensor([self.batch_size], self.type_sequence_label_size) |
|
token_labels = ids_tensor([self.batch_size, self.seq_length], self.num_labels) |
|
choice_labels = ids_tensor([self.batch_size], self.num_choices) |
|
|
|
config = AlbertConfig( |
|
vocab_size=self.vocab_size, |
|
hidden_size=self.hidden_size, |
|
num_hidden_layers=self.num_hidden_layers, |
|
num_attention_heads=self.num_attention_heads, |
|
intermediate_size=self.intermediate_size, |
|
hidden_act=self.hidden_act, |
|
hidden_dropout_prob=self.hidden_dropout_prob, |
|
attention_probs_dropout_prob=self.attention_probs_dropout_prob, |
|
max_position_embeddings=self.max_position_embeddings, |
|
type_vocab_size=self.type_vocab_size, |
|
initializer_range=self.initializer_range, |
|
num_hidden_groups=self.num_hidden_groups, |
|
) |
|
|
|
return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels |
|
|
|
def create_and_check_model( |
|
self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels |
|
): |
|
model = AlbertModel(config=config) |
|
model.to(torch_device) |
|
model.eval() |
|
result = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids) |
|
result = model(input_ids, token_type_ids=token_type_ids) |
|
result = model(input_ids) |
|
self.parent.assertEqual(result.last_hidden_state.shape, (self.batch_size, self.seq_length, self.hidden_size)) |
|
self.parent.assertEqual(result.pooler_output.shape, (self.batch_size, self.hidden_size)) |
|
|
|
def create_and_check_for_pretraining( |
|
self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels |
|
): |
|
model = AlbertForPreTraining(config=config) |
|
model.to(torch_device) |
|
model.eval() |
|
result = model( |
|
input_ids, |
|
attention_mask=input_mask, |
|
token_type_ids=token_type_ids, |
|
labels=token_labels, |
|
sentence_order_label=sequence_labels, |
|
) |
|
self.parent.assertEqual(result.prediction_logits.shape, (self.batch_size, self.seq_length, self.vocab_size)) |
|
self.parent.assertEqual(result.sop_logits.shape, (self.batch_size, config.num_labels)) |
|
|
|
def create_and_check_for_masked_lm( |
|
self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels |
|
): |
|
model = AlbertForMaskedLM(config=config) |
|
model.to(torch_device) |
|
model.eval() |
|
result = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, labels=token_labels) |
|
self.parent.assertEqual(result.logits.shape, (self.batch_size, self.seq_length, self.vocab_size)) |
|
|
|
def create_and_check_for_question_answering( |
|
self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels |
|
): |
|
model = AlbertForQuestionAnswering(config=config) |
|
model.to(torch_device) |
|
model.eval() |
|
result = model( |
|
input_ids, |
|
attention_mask=input_mask, |
|
token_type_ids=token_type_ids, |
|
start_positions=sequence_labels, |
|
end_positions=sequence_labels, |
|
) |
|
self.parent.assertEqual(result.start_logits.shape, (self.batch_size, self.seq_length)) |
|
self.parent.assertEqual(result.end_logits.shape, (self.batch_size, self.seq_length)) |
|
|
|
def create_and_check_for_sequence_classification( |
|
self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels |
|
): |
|
config.num_labels = self.num_labels |
|
model = AlbertForSequenceClassification(config) |
|
model.to(torch_device) |
|
model.eval() |
|
result = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, labels=sequence_labels) |
|
self.parent.assertEqual(result.logits.shape, (self.batch_size, self.num_labels)) |
|
|
|
def create_and_check_for_token_classification( |
|
self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels |
|
): |
|
config.num_labels = self.num_labels |
|
model = AlbertForTokenClassification(config=config) |
|
model.to(torch_device) |
|
model.eval() |
|
result = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, labels=token_labels) |
|
self.parent.assertEqual(result.logits.shape, (self.batch_size, self.seq_length, self.num_labels)) |
|
|
|
def create_and_check_for_multiple_choice( |
|
self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels |
|
): |
|
config.num_choices = self.num_choices |
|
model = AlbertForMultipleChoice(config=config) |
|
model.to(torch_device) |
|
model.eval() |
|
multiple_choice_inputs_ids = input_ids.unsqueeze(1).expand(-1, self.num_choices, -1).contiguous() |
|
multiple_choice_token_type_ids = token_type_ids.unsqueeze(1).expand(-1, self.num_choices, -1).contiguous() |
|
multiple_choice_input_mask = input_mask.unsqueeze(1).expand(-1, self.num_choices, -1).contiguous() |
|
result = model( |
|
multiple_choice_inputs_ids, |
|
attention_mask=multiple_choice_input_mask, |
|
token_type_ids=multiple_choice_token_type_ids, |
|
labels=choice_labels, |
|
) |
|
self.parent.assertEqual(result.logits.shape, (self.batch_size, self.num_choices)) |
|
|
|
def prepare_config_and_inputs_for_common(self): |
|
config_and_inputs = self.prepare_config_and_inputs() |
|
( |
|
config, |
|
input_ids, |
|
token_type_ids, |
|
input_mask, |
|
sequence_labels, |
|
token_labels, |
|
choice_labels, |
|
) = config_and_inputs |
|
inputs_dict = {"input_ids": input_ids, "token_type_ids": token_type_ids, "attention_mask": input_mask} |
|
return config, inputs_dict |
|
|
|
|
|
@require_torch |
|
class AlbertModelTest(ModelTesterMixin, unittest.TestCase): |
|
|
|
all_model_classes = ( |
|
( |
|
AlbertModel, |
|
AlbertForPreTraining, |
|
AlbertForMaskedLM, |
|
AlbertForMultipleChoice, |
|
AlbertForSequenceClassification, |
|
AlbertForTokenClassification, |
|
AlbertForQuestionAnswering, |
|
) |
|
if is_torch_available() |
|
else () |
|
) |
|
fx_ready_model_classes = all_model_classes |
|
|
|
test_sequence_classification_problem_types = True |
|
|
|
|
|
def _prepare_for_class(self, inputs_dict, model_class, return_labels=False): |
|
inputs_dict = super()._prepare_for_class(inputs_dict, model_class, return_labels=return_labels) |
|
|
|
if return_labels: |
|
if model_class in get_values(MODEL_FOR_PRETRAINING_MAPPING): |
|
inputs_dict["labels"] = torch.zeros( |
|
(self.model_tester.batch_size, self.model_tester.seq_length), dtype=torch.long, device=torch_device |
|
) |
|
inputs_dict["sentence_order_label"] = torch.zeros( |
|
self.model_tester.batch_size, dtype=torch.long, device=torch_device |
|
) |
|
return inputs_dict |
|
|
|
def setUp(self): |
|
self.model_tester = AlbertModelTester(self) |
|
self.config_tester = ConfigTester(self, config_class=AlbertConfig, hidden_size=37) |
|
|
|
def test_config(self): |
|
self.config_tester.run_common_tests() |
|
|
|
def test_model(self): |
|
config_and_inputs = self.model_tester.prepare_config_and_inputs() |
|
self.model_tester.create_and_check_model(*config_and_inputs) |
|
|
|
def test_for_pretraining(self): |
|
config_and_inputs = self.model_tester.prepare_config_and_inputs() |
|
self.model_tester.create_and_check_for_pretraining(*config_and_inputs) |
|
|
|
def test_for_masked_lm(self): |
|
config_and_inputs = self.model_tester.prepare_config_and_inputs() |
|
self.model_tester.create_and_check_for_masked_lm(*config_and_inputs) |
|
|
|
def test_for_multiple_choice(self): |
|
config_and_inputs = self.model_tester.prepare_config_and_inputs() |
|
self.model_tester.create_and_check_for_multiple_choice(*config_and_inputs) |
|
|
|
def test_for_question_answering(self): |
|
config_and_inputs = self.model_tester.prepare_config_and_inputs() |
|
self.model_tester.create_and_check_for_question_answering(*config_and_inputs) |
|
|
|
def test_for_sequence_classification(self): |
|
config_and_inputs = self.model_tester.prepare_config_and_inputs() |
|
self.model_tester.create_and_check_for_sequence_classification(*config_and_inputs) |
|
|
|
def test_model_various_embeddings(self): |
|
config_and_inputs = self.model_tester.prepare_config_and_inputs() |
|
for type in ["absolute", "relative_key", "relative_key_query"]: |
|
config_and_inputs[0].position_embedding_type = type |
|
self.model_tester.create_and_check_model(*config_and_inputs) |
|
|
|
@slow |
|
def test_model_from_pretrained(self): |
|
for model_name in ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]: |
|
model = AlbertModel.from_pretrained(model_name) |
|
self.assertIsNotNone(model) |
|
|
|
|
|
@require_torch |
|
class AlbertModelIntegrationTest(unittest.TestCase): |
|
@slow |
|
def test_inference_no_head_absolute_embedding(self): |
|
model = AlbertModel.from_pretrained("albert-base-v2") |
|
input_ids = torch.tensor([[0, 345, 232, 328, 740, 140, 1695, 69, 6078, 1588, 2]]) |
|
attention_mask = torch.tensor([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]) |
|
output = model(input_ids, attention_mask=attention_mask)[0] |
|
expected_shape = torch.Size((1, 11, 768)) |
|
self.assertEqual(output.shape, expected_shape) |
|
expected_slice = torch.tensor( |
|
[[[-0.6513, 1.5035, -0.2766], [-0.6515, 1.5046, -0.2780], [-0.6512, 1.5049, -0.2784]]] |
|
) |
|
|
|
self.assertTrue(torch.allclose(output[:, 1:4, 1:4], expected_slice, atol=1e-4)) |
|
|