Kaggle competition: Tweet Sentiment Extraction
<SOS> metric: Jaccard Kaggle data source: tweet-sentiment-extraction<set_options>
import numpy as np
import pandas as pd
import os
import re
import string
import tokenizers
import transformers
import torch
import torch.nn as nn
from sklearn import model_selection
from torch.nn import functional as F
from tqdm import tqdm
from transformers import AdamW
from transformers import get_linear_schedule_with_warmup
pd.set_option('display.max_columns', None)
 pd.set_option('display.max_rows', None)
 test_df.head()<concatenate>
MAX_LEN = 192
TEST_BATCH_SIZE = 32
EPOCHS = 5
ROBERTA_PATH = ".. /input/roberta-base"
TOKENIZER = tokenizers.ByteLevelBPETokenizer(
 vocab_file=f"{ROBERTA_PATH}/vocab.json",
merges_file=f"{ROBERTA_PATH}/merges.txt",
lowercase=True,
add_prefix_space=True 
)
 DEVICE = 'cuda'
TEST_PATH = '.. /input/tweet-sentiment-extraction/test.csv'
FOLDS = 5
df_combine=pd.concat([df,test_df],axis=0, sort = False,ignore_index = True)
 df_combine.shape<define_variables>
test_df = pd.read_csv(TEST_PATH)
 test_df
ordinal_catg_col = ["Education", "EnvironmentSatisfaction", "JobInvolvement","JobSatisfaction",
"PerformanceRating","StockOptionLevel","CommunicationSkill", "Behaviour" ]
print(ordinal_catg_col )<categorify>
class AverageMeter:

 def __init__(self):
self.reset()
def reset(self):
self.val = 0
self.avg = 0
self.sum = 0
self.count = 0
def update(self, val, n=1):
self.val = val
self.sum += val * n
self.count += n
self.avg = self.sum / self.count
class EarlyStopping:
def __init__(self, patience=7, mode="max", delta=0.001):
self.patience = patience
self.counter = 0
self.mode = mode
self.best_score = None
self.early_stop = False
self.delta = delta
if self.mode == "min":
self.val_score = np.Inf
else:
self.val_score = -np.Inf
def __call__(self, epoch_score, model, model_path):
if self.mode == "min":
score = -1.0 * epoch_score
else:
score = np.copy(epoch_score)
 if self.best_score is None:
self.best_score = score
self.save_checkpoint(epoch_score, model, model_path)
 elif score < self.best_score + self.delta:
self.counter += 1
print('EarlyStopping counter: {} out of {}'.format(self.counter, self.patience))
if self.counter >= self.patience:
self.early_stop = True
else:
self.best_score = score
self.save_checkpoint(epoch_score, model, model_path)
 self.counter = 0
def save_checkpoint(self, epoch_score, model, model_path):
if epoch_score not in [-np.inf, np.inf, -np.nan, np.nan]:
print('Validation score improved({} --> {} ).Saving model!'.format(self.val_score, epoch_score))
torch.save(model.state_dict() , model_path)
 self.val_score = epoch_score
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))
df_BusinessTravel_one_hot = pd.get_dummies(df_combine['BusinessTravel'], prefix='BusinessTravel')
 df_Department_one_hot = pd.get_dummies(df_combine['Department'], prefix='Department')
 df_EducationField_one_hot = pd.get_dummies(df_combine['EducationField'], prefix='EducationField')
 df_Gender_one_hot = pd.get_dummies(df_combine['Gender'], prefix='Gender',drop_first=True)
 df_JobRole_one_hot = pd.get_dummies(df_combine['JobRole'], prefix='JobRole')
 df_MaritalStatus_one_hot = pd.get_dummies(df_combine['MaritalStatus'], prefix='MaritalStatus')
 df_OverTime_one_hot = pd.get_dummies(df_combine['OverTime'], prefix='OverTime',drop_first=True )<concatenate>
class TweetModel(transformers.BertPreTrainedModel):
def __init__(self, conf):
super(TweetModel, self ).__init__(conf)
 config = transformers.RobertaConfig.from_pretrained(
 '.. /input/roberta-base/config.json', output_hidden_states=True)
 self.roberta = transformers.RobertaModel.from_pretrained(
 '.. /input/roberta-base/pytorch_model.bin', config=config)
 self.drop_out = nn.Dropout(0.3)
 self.fc = nn.Linear(768, 2)
 nn.init.normal_(self.fc.weight, std=0.02)
 nn.init.normal_(self.fc.bias, 0)
 def forward(self, ids, mask):
_, _, out = self.roberta(
 ids,
attention_mask = mask 
)
 out = torch.stack([out[-1], out[-2], out[-3], out[-4]])
 out = torch.mean(out, 0)
 out = self.drop_out(out)
 logits = self.fc(out)
 start_logits, end_logits = logits.split(1, dim = -1)
 start_logits = start_logits.squeeze(-1)
 end_logits = end_logits.squeeze(-1)
 return start_logits, end_logits
df_nominal_catg = pd.concat([df_BusinessTravel_one_hot, df_Department_one_hot, df_EducationField_one_hot,
df_Gender_one_hot, df_JobRole_one_hot, df_MaritalStatus_one_hot,
df_OverTime_one_hot], axis=1)
 print(df_nominal_catg.shape)
 df_nominal_catg.head()<drop_column>
class TweetDataset:
def __init__(self, tweet, sentiment):
self.tweet = tweet
self.sentiment = sentiment
self.tokenizer = TOKENIZER
self.max_len = MAX_LEN
def __len__(self):
return len(self.tweet)
 def __getitem__(self, item):
data = process_data(
 self.tweet[item],
self.sentiment[item],
self.tokenizer,
self.max_len 
)
 return {
'ids': torch.tensor(data['ids'], dtype = torch.long),
'mask': torch.tensor(data['mask'], dtype = torch.long),
'token_type_ids': torch.tensor(data['token_type_ids'], dtype = torch.long),
'orig_tweet': data['orig_tweet'],
'sentiment': data['sentiment'],
'offsets': torch.tensor(data['offsets'], dtype = torch.long)
 }
final_catg_col = nominal_catg_col + ordinal_catg_col + ["Attrition"]
df_numeric = df_combine.drop(final_catg_col , axis=1)
 df_numeric.shape<normalization>
def calculate_jaccard_score(
 original_tweet,
sentiment_val,
idx_start,
idx_end,
offsets,
verbose=False):
if idx_end < idx_start:
idx_end = idx_start
filtered_output = ""
if sentiment_val == "neutral" or len(original_tweet.split())< 2:
filtered_output = original_tweet
else:
for ix in range(idx_start, idx_end + 1):
filtered_output += original_tweet[offsets[ix][0]: offsets[ix][1]]
if(ix+1)< len(offsets)and offsets[ix][1] < offsets[ix+1][0]:
filtered_output += " "
return filtered_output
numeric_col = list(df_numeric.columns)
 scaler = MinMaxScaler(feature_range=(0, 1))
df_numeric = scaler.fit_transform(df_numeric)
 df_numeric = pd.DataFrame(df_numeric, columns= numeric_col )<concatenate>
model_config = transformers.RobertaConfig.from_pretrained(ROBERTA_PATH)
 model_config.output_hidden_states = True
model1 = TweetModel(conf=model_config)
 model1.to(DEVICE)
 model1.load_state_dict(torch.load(".. /input/roberta-sentiment-extraction/model_0.bin"))
model1.eval()
model2 = TweetModel(conf=model_config)
 model2.to(DEVICE)
 model2.load_state_dict(torch.load(".. /input/roberta-sentiment-extraction/model_1.bin"))
model2.eval()
model3 = TweetModel(conf=model_config)
 model3.to(DEVICE)
 model3.load_state_dict(torch.load(".. /input/roberta-sentiment-extraction/model_2.bin"))
model3.eval()
model4 = TweetModel(conf=model_config)
 model4.to(DEVICE)
 model4.load_state_dict(torch.load(".. /input/roberta-sentiment-extraction/model_3.bin"))
model4.eval()
model5 = TweetModel(conf=model_config)
 model5.to(DEVICE)
 model5.load_state_dict(torch.load(".. /input/roberta-sentiment-extraction/model_4.bin"))
model5.eval()
df_pre_process = pd.concat([df_numeric, df_ordinal_catg.reset_index(drop=True),
df_nominal_catg.reset_index(drop=True)],axis=1,sort = False)
 print(df_pre_process.shape)
 df_pre_process.head()<count_values>
def eval_fn(data_loader, device):
final_outputs = []
with torch.no_grad() :
tk0 = tqdm(data_loader, total = len(data_loader))
for bi, d in enumerate(tk0):
ids = d["ids"]
mask = d["mask"]
token_type_ids = d["token_type_ids"]
orig_tweet = d["orig_tweet"]
sentiment = d["sentiment"]
offsets = d["offsets"]
ids = ids.to(device, dtype=torch.long)
 token_type_ids = token_type_ids.to(device, dtype=torch.long)
 mask = mask.to(device, dtype=torch.long)
 predicted_start1, predicted_end1 = model1(ids, mask)
 predicted_start2, predicted_end2 = model2(ids, mask)
 predicted_start3, predicted_end3 = model3(ids, mask)
 predicted_start4, predicted_end4 = model4(ids, mask)
 predicted_start5, predicted_end5 = model5(ids, mask)
 predicted_start =(predicted_start1 + predicted_start2 + predicted_start3 + predicted_start4 + predicted_start5)/ 5
predicted_end =(predicted_end1 + predicted_end2 + predicted_end3 + predicted_end4 + predicted_end5)/ 5
predicted_start = torch.softmax(predicted_start, dim = 1 ).cpu().detach().numpy()
predicted_end = torch.softmax(predicted_end, dim = 1 ).cpu().detach().numpy()
for i, tweet in enumerate(orig_tweet):
tweet_sentiment = sentiment[i]
output_sentence = calculate_jaccard_score(
 original_tweet = tweet,
sentiment_val = tweet_sentiment,
idx_start = np.argmax(predicted_start[i, :]),
idx_end = np.argmax(predicted_end[i, :]),
offsets = offsets[i] 
)
 final_outputs.append(output_sentence)
 return final_outputs
df_age = feat_to_binning("Age",5)
 print(df_age['Age_bin_round'].value_counts())
 df_age.head()<prepare_x_and_y>
def predict() :
df_test = test_df
test_dataset = TweetDataset(
 tweet = df_test.text.values,
sentiment = df_test.sentiment.values 
)
 test_dataloader = torch.utils.data.DataLoader(
 test_dataset,
batch_size = TEST_BATCH_SIZE,
num_workers = 0 
)
 outputs = eval_fn(test_dataloader, DEVICE)
 sample = pd.read_csv(".. /input/tweet-sentiment-extraction/sample_submission.csv")
 sample.loc[:, 'selected_text'] = outputs
sample.to_csv("submission.csv", index=False )
<create_dataframe><EOS> 
predict()