Kaggle competition: Tweet Sentiment Extraction
a = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv')
 b = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv' )<set_options>
from torch import nn
import torch
from torch.utils.data import Dataset, DataLoader
import transformers
from transformers.optimization import AdamW
import tokenizers
from wordcloud import WordCloud, STOPWORDS
import seaborn as sns
import matplotlib.pyplot as plt
pd.set_option('display.max_columns', None)
 pd.set_option('display.max_rows', None )<train_model>
basePath = '/kaggle/input/tweet-sentiment-extraction/'
raw_train_data = pd.read_csv(basePath+'train.csv')
 raw_train_data.head(15 )
print('The shape of our training set: %s houses and %s features'%(a.shape[0],a.shape[1]))
print('The shape of our training set: %s houses and %s features'%(b.shape[0],b.shape[1]))
print('The testing set has 1 feature less than the training set, which is SalePrice, the target to predict ' )<prepare_x_and_y>
class config:
BERT_PATH = ".. /input/bert-base-uncased/"
roberta_path = ".. /input/roberta-base/"
na = a.shape[0]
nb = b.shape[0]
y_train = a['SalePrice'].to_frame()
c1 = pd.concat(( a, b), sort=False ).reset_index(drop=True)
 c1.drop(['SalePrice'], axis=1, inplace=True)
 c1.drop(['Id'], axis=1, inplace=True)
 print(f"Total size is {c1.shape}" )<compute_test_metric>
MAX_SEQ_LENGTH = 190
TRAIN_BATCH_SIZE = 16
EVAL_BATCH_SIZE = 16
TEST_BATCH_SIZE = 16
LEARNING_RATE = 1e-5
NUM_TRAIN_EPOCHS = 3
BERT_TYPE = "roberta-base-uncased"
max_grad_norm = 1.0
vocab_file = config.roberta_path + "vocab.json"
merge_file = config.roberta_path + "merges.txt"
msv1(c1, 20, color=sns.color_palette('Reds',15))<drop_column>
tokenizer = tokenizers.ByteLevelBPETokenizer(vocab_file=vocab_file,merges_file=merge_file,lowercase=True,add_prefix_space=True )
c=c1.dropna(thresh=len(c1)*0.8, axis=1)
 print(f"We dropped {c1.shape[1]-c.shape[1]} features in the combined set" )<data_type_conversions>
raw_train_data = raw_train_data.dropna()
NA=c[allna.index.to_list() ]<data_type_conversions>
print('The number of training data points are: ',raw_train_data.shape[0] )
c['MasVnrArea']=c.MasVnrArea.fillna(0)
 c['LotFrontage']=c.LotFrontage.fillna(c.LotFrontage.median())
 c['GarageYrBlt']=c["GarageYrBlt"].fillna(1980 )<categorify>
raw_test_data = pd.read_csv(basePath+'test.csv')
 raw_test_data.head(10 )
ffill_cols = ['Electrical', 'SaleType', 'KitchenQual', 'Exterior1st',
'Exterior2nd', 'Functional', 'Utilities', 'MSZoning']
def filling_NA(data, columns, METHOD='ffill'):
fill_cols = columns
for col in data[fill_cols]:
data[col]= data[col].fillna(method=METHOD)
 return data<compute_test_metric>
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))
msv2(c )<concatenate>
def loss_func(out, s_target, e_target):
criterion = nn.CrossEntropyLoss()
s_loss = criterion(out[0], s_target)
 e_loss = criterion(out[1], e_target)
 total_loss = s_loss+e_loss
return total_loss
d=filling_NA(c, ffill_cols )<data_type_conversions>
class BertBaseQA(nn.Module):
def __init__(self, hidden_size, num_labels,conf):
super().__init__()
self.hidden_size = hidden_size
self.num_labels = num_labels
self.bert = transformers.RobertaModel.from_pretrained(config.roberta_path,config=conf)
 self.drop_out = nn.Dropout(0.1)
 self.qa_outputs = nn.Linear(self.hidden_size, self.num_labels)
 def forward(self, ids, mask, token_ids):
output = self.bert(
 input_ids = ids,
attention_mask = mask,
token_type_ids = token_ids, 
)
 sequence_output = output[0]
sequence_output = self.drop_out(sequence_output)
 logits = self.qa_outputs(sequence_output)
 start_logits, end_logits = logits.split(1, dim=-1)
 start_logits = start_logits.squeeze(-1)
 end_logits = end_logits.squeeze(-1)
 outputs =(start_logits, end_logits,)
 return outputs
fill_cols = ['Electrical', 'SaleType', 'KitchenQual', 'Exterior1st',
'Exterior2nd', 'Functional', 'Utilities', 'MSZoning']
for col in c[fill_cols]:
c[col] = c[col].fillna(method='ffill' )<data_type_conversions>
class BertDatasetModule(Dataset):
def __init__(self, tokenizer, context, question, max_length, text):
self.context = context
self.question = question
self.text = text
self.tokenizer = tokenizer
self.max_length = max_length
def __len__(self):
'Denotes the total number of samples'
return len(self.context)
 def __getitem__(self, idx):
'Generates one sample of data'
context_ = self.context[idx]
question_ = self.question[idx]
text_ = self.text[idx]
context_ = " " + " ".join(str(context_ ).split())
 text_ = " " + " ".join(str(text_ ).split())
 tok_context = tokenizer.encode(context_)
 tok_question = tokenizer.encode(question_)
 tok_answer = tokenizer.encode(text_)
 context_ids = tok_context.ids
question_ids = tok_question.ids
answer_ids = tok_answer.ids
offsets_orig = tok_context.offsets
input_ids = [0] + question_ids + [2] + [2] + context_ids + [2]
token_type_ids = [0, 0, 0, 0] + [0] *(len(context_ids)) + [0]
mask = [1] * len(token_type_ids)
 offsets = [(0,0)]*4 + offsets_orig + [(0,0)]
s_pos, e_pos = 0, 0
for i in range(len(input_ids)) :
if(input_ids[i: i+len(answer_ids)] == answer_ids):
s_pos = i
e_pos = i + len(answer_ids)- 1
break
assert(( s_pos<len(input_ids)) &(e_pos<len(input_ids)) &(s_pos<=e_pos))
pad_length = self.max_length - len(input_ids)
 if(len(input_ids)<self.max_length):
ids = input_ids +([1]*pad_length)
 elif(len(input_ids)>self.max_length):
ids = input_ids[:self.max_length]
if(len(token_type_ids)<self.max_length):
token_ids = token_type_ids +([0]*pad_length)
 elif(len(token_type_ids)>self.max_length):
token_ids = token_type_ids[:self.max_length]
if(len(mask)<self.max_length):
mask_ids = mask +([0]*pad_length)
 elif(len(token_type_ids)>self.max_length):
mask_ids = mask[:self.max_length]
if(len(input_ids)<self.max_length):
offsets = offsets +([(0, 0)] * pad_length)
 elif(len(input_ids)>self.max_length):
offsets = offsets[:self.max_length]
ids = torch.tensor(ids, dtype = torch.long)
 tt_ids = torch.tensor(token_ids, dtype = torch.long)
 mask_ids = torch.tensor(mask_ids, dtype = torch.long)
 offsets = torch.tensor(offsets, dtype = torch.long)
 start_pos = torch.tensor(s_pos, dtype = torch.long)
 end_pos = torch.tensor(e_pos, dtype = torch.long)
 return {'ids': ids,
'token_type_ids': tt_ids,
'mask':mask_ids,
'start_pos': start_pos,
'end_pos': end_pos,
'offsets': offsets,
'context':context_,
'question':question_,
'text':text_}
NAcols=c.columns
for col in NAcols:
if c[col].dtype == "object":
c[col] = c[col].fillna("None" )<data_type_conversions>
def train_loop(dataloader, model, optimizer, device, max_grad_norm, scheduler=None):
model.train()
for bi, d in enumerate(tqdm_notebook(dataloader, desc="Iteration")) :
ids = d['ids']
mask_ids = d['mask']
token_ids = d['token_type_ids']
start_pos = d['start_pos']
end_pos = d['end_pos']
offsets = d['offsets']
ids = ids.to(device, dtype = torch.long)
 mask_ids = mask_ids.to(device, dtype = torch.long)
 token_ids = token_ids.to(device, dtype = torch.long)
 start_pos = start_pos.to(device, dtype = torch.long)
 end_pos = end_pos.to(device, dtype = torch.long)
 optimizer.zero_grad()
start_and_end_scores = model(ids, mask_ids, token_ids)
 loss = loss_func(start_and_end_scores, start_pos, end_pos)
 loss.backward()
optimizer.step()
if scheduler is not None:
scheduler.step()
if bi%100==0:
print(f"bi: {bi}, loss: {loss}" )
for col in NAcols:
if c[col].dtype != "object":
c[col]= c[col].fillna(0 )<feature_engineering>
def eval_loop(dataloader, model, device):
model.eval()
pred_s = None
pred_e = None
eval_loss = 0.0
eval_steps = 0
for bi, d in enumerate(dataloader):
ids = d['ids']
mask_ids = d['mask']
token_ids = d['token_type_ids']
start_pos = d['start_pos']
end_pos = d['end_pos']
context = d['context']
question = d['question']
selected_text = d['text']
offsets = d['offsets']
ids = ids.to(device, dtype = torch.long)
 mask_ids = mask_ids.to(device, dtype = torch.long)
 token_ids = token_ids.to(device, dtype = torch.long)
 start_pos = start_pos.to(device, dtype = torch.long)
 end_pos = end_pos.to(device, dtype = torch.long)
 with torch.no_grad() :
start_and_end_scores = model(ids,mask_ids, token_ids)
 loss = loss_func(start_and_end_scores, start_pos, end_pos)
 eval_loss += loss.mean().item()
eval_steps += 1
pred_s = torch.softmax(torch.tensor(start_and_end_scores[0]),dim=1 ).detach().cpu().numpy()
pred_e = torch.softmax(torch.tensor(start_and_end_scores[1]),dim=1 ).detach().cpu().numpy()
eval_loss = eval_loss/eval_steps
pred_start = np.argmax(pred_s, axis=1)
 pred_end = np.argmax(pred_e, axis=1)
 jaccards=[]
for i,tweet in enumerate(context):
idx_start = pred_start[i]
idx_end = pred_end[i]
if idx_end < idx_start:
idx_end = idx_start
filtered_output = ""
for ix in range(idx_start, idx_end + 1):
filtered_output += tweet[offsets[i][ix][0]: offsets[i][ix][1]]
if(ix+1)< len(offsets[i])and offsets[i][ix][1] < offsets[i][ix+1][0]:
filtered_output += " "
if question[i] == "neutral" or len(tweet.split())< 2:
filtered_output = tweet
jaccards.append(jaccard(filtered_output,selected_text[i]))
return eval_loss, pred_start, pred_end, jaccards
c['TotalArea'] = c['TotalBsmtSF'] + c['1stFlrSF'] + c['2ndFlrSF'] + c['GrLivArea'] +c['GarageArea']
c['Bathrooms'] = c['FullBath'] + c['HalfBath']*0.5
c['Year average']=(c['YearRemodAdd']+c['YearBuilt'])/2<data_type_conversions>
test_start = []
test_end = []
mean_jaccard = []
c['MSSubClass'] = c['MSSubClass'].apply(str)
 c['YrSold'] = c['YrSold'].astype(str )<categorify>
kf = StratifiedKFold(n_splits = 5, shuffle = True, random_state = 42)
 for fold,(tr_ind, val_ind)in enumerate(kf.split(raw_train_data, raw_train_data['sentiment'])) :
train_data = raw_train_data.iloc[tr_ind].reset_index(drop=True)
 val_data = raw_train_data.iloc[val_ind].reset_index(drop=True)
 train_dataset = BertDatasetModule(
 tokenizer = tokenizer,
context = train_data['text'],
question = train_data['sentiment'],
max_length = MAX_SEQ_LENGTH,
text = train_data['selected_text'] 
)
 train_dataloader = DataLoader(train_dataset, batch_size = TRAIN_BATCH_SIZE, shuffle=True)
 eval_dataset = BertDatasetModule(
 tokenizer = tokenizer,
context = val_data['text'],
question = val_data['sentiment'],
max_length = MAX_SEQ_LENGTH,
text = val_data['selected_text'] 
)
 eval_dataloader = DataLoader(eval_dataset, batch_size = EVAL_BATCH_SIZE, shuffle=False)
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 print(device)
 model_config = transformers.RobertaConfig.from_pretrained(config.roberta_path)
 model = BertBaseQA(768, 2,model_config ).to(device)
 optimizer = AdamW(model.parameters() , lr=LEARNING_RATE, correct_bias=False)
 NUM_TRAIN_STEPS = int(len(train_dataset)/TRAIN_BATCH_SIZE * NUM_TRAIN_EPOCHS)
 scheduler = transformers.get_constant_schedule_with_warmup(
 optimizer,
num_warmup_steps=500,
last_epoch=-1)
 for epoch in trange(NUM_TRAIN_EPOCHS):
train_loop(train_dataloader, model, optimizer, device, max_grad_norm, scheduler)
 res = eval_loop(eval_dataloader, model, device)
 val_start = res[1]
val_end = res[2]
jaccards = res[3]
mean_jaccard.append(np.mean(jaccards))
torch.save(model.state_dict() ,'model_' + str(fold)+ '.pth' )
cb=pd.get_dummies(c)
 print(f"the shape of the original dataset {c.shape}")
 print(f"the shape of the encoded dataset {cb.shape}")
 print(f"We have {cb.shape[1]- c.shape[1]} new encoded features" )<split>
predicted_output = []
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 model_config = transformers.RobertaConfig.from_pretrained(config.roberta_path)
 model_0 = BertBaseQA(768, 2,model_config ).to(device)
 model_1 = BertBaseQA(768, 2,model_config ).to(device)
 model_2 = BertBaseQA(768, 2,model_config ).to(device)
 model_3 = BertBaseQA(768, 2,model_config ).to(device)
 model_4 = BertBaseQA(768, 2,model_config ).to(device)
 model_0.load_state_dict(torch.load('model_0.pth'))
model_1.load_state_dict(torch.load('model_1.pth'))
model_2.load_state_dict(torch.load('model_2.pth'))
model_3.load_state_dict(torch.load('model_3.pth'))
model_4.load_state_dict(torch.load('model_4.pth'))
model_0.eval()
model_1.eval()
model_2.eval()
model_3.eval()
model_4.eval()
test_dataset = BertDatasetModule(
 tokenizer = tokenizer,
context = raw_test_data['text'],
question = raw_test_data['sentiment'],
max_length = MAX_SEQ_LENGTH,
text = raw_test_data['text'] 
)
 test_dataloader = DataLoader(test_dataset, batch_size = TEST_BATCH_SIZE, shuffle=False)
 for bi, d in enumerate(test_dataloader):
ids = d['ids']
mask_ids = d['mask']
token_ids = d['token_type_ids']
start_pos = d['start_pos']
end_pos = d['end_pos']
context = d['context']
question = d['question']
selected_text = d['text']
offsets = d['offsets']
ids = ids.to(device, dtype = torch.long)
 mask_ids = mask_ids.to(device, dtype = torch.long)
 token_ids = token_ids.to(device, dtype = torch.long)
 start_pos = start_pos.to(device, dtype = torch.long)
 end_pos = end_pos.to(device, dtype = torch.long)
 with torch.no_grad() :
start_and_end_scores0 = model_0(ids,mask_ids, token_ids)
 start_and_end_scores1 = model_1(ids,mask_ids, token_ids)
 start_and_end_scores2 = model_2(ids,mask_ids, token_ids)
 start_and_end_scores3 = model_2(ids,mask_ids, token_ids)
 start_and_end_scores4 = model_3(ids,mask_ids, token_ids)
 start_scores =(start_and_end_scores0[0]+start_and_end_scores1[0]+start_and_end_scores2[0]+start_and_end_scores3[0]+start_and_end_scores4[0])/5
end_scores =(start_and_end_scores0[1]+start_and_end_scores1[1]+start_and_end_scores2[1]+start_and_end_scores3[1]+start_and_end_scores4[1])/5
pred_s = torch.softmax(torch.tensor(start_scores),dim=1 ).detach().cpu().numpy()
pred_e = torch.softmax(torch.tensor(end_scores),dim=1 ).detach().cpu().numpy()
pred_start = np.argmax(pred_s, axis=1)
 pred_end = np.argmax(pred_e, axis=1)
 for i,tweet in enumerate(context):
idx_start = pred_start[i]
idx_end = pred_end[i]
if idx_end < idx_start:
idx_end = idx_start
filtered_output = ""
for ix in range(idx_start, idx_end + 1):
filtered_output += tweet[offsets[i][ix][0]: offsets[i][ix][1]]
if(ix+1)< len(offsets[i])and offsets[i][ix][1] < offsets[i][ix+1][0]:
filtered_output += " "
if question == "neutral" or len(tweet.split())< 2:
filtered_output = tweet
predicted_output.append(filtered_output )
<sort_values><EOS> 
sample = pd.read_csv(".. /input/tweet-sentiment-extraction/sample_submission.csv")
 sample.loc[:, 'selected_text'] = predicted_output
sample.to_csv("submission.csv", index=False)
 sample.head(30 )