File size: 2,380 Bytes
2dcbc14 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 |
# Let's import a few requirements
import torch
from transformers import AutoModelForSequenceClassification, AutoTokenizer
import numpy
class TransformerVectorizer:
def __init__(self):
# Load the tokenizer (converts text to tokens)
self.tokenizer = AutoTokenizer.from_pretrained("cardiffnlp/twitter-roberta-base-sentiment-latest")
# Load the pre-trained model
self.transformer_model = AutoModelForSequenceClassification.from_pretrained(
"cardiffnlp/twitter-roberta-base-sentiment-latest"
)
self.device = "cuda:0" if torch.cuda.is_available() else "cpu"
def text_to_tensor(
self,
texts: list,
) -> numpy.ndarray:
"""Function that transforms a list of texts to their learned representation.
Args:
list_text_X (list): List of texts to be transformed.
Returns:
numpy.ndarray: Transformed list of texts.
"""
# First, tokenize all the input text
tokenized_text_X_train = self.tokenizer.batch_encode_plus(
texts, return_tensors="pt"
)["input_ids"]
# Depending on the hardware used, the number of examples to be processed can be reduced
# Here we split the data into 100 examples per batch
tokenized_text_X_train_split = torch.split(tokenized_text_X_train, split_size_or_sections=50)
# Send the model to the device
transformer_model = self.transformer_model.to(self.device)
output_hidden_states_list = []
for tokenized_x in tokenized_text_X_train_split:
# Pass the tokens through the transformer model and get the hidden states
# Only keep the last hidden layer state for now
output_hidden_states = transformer_model(tokenized_x.to(self.device), output_hidden_states=True)[
1
][-1]
# Average over the tokens axis to get a representation at the text level.
output_hidden_states = output_hidden_states.mean(dim=1)
output_hidden_states = output_hidden_states.detach().cpu().numpy()
output_hidden_states_list.append(output_hidden_states)
self.encodings = numpy.concatenate(output_hidden_states_list, axis=0)
return self.encodings
def transform(self, texts: list):
return self.text_to_tensor(texts)
|