Kaggle competition: Tweet Sentiment Extraction
<SOS> metric: Jaccard Kaggle data source: tweet-sentiment-extraction<groupby>
import numpy as np
import pandas as pd
import os
import tokenizers
import string
import torch
import transformers
import torch.nn as nn
from torch.nn import functional as F
from tqdm.autonotebook import tqdm
import re
from torch.optim import lr_scheduler
from sklearn import model_selection
from sklearn import metrics
from transformers import AdamW
from transformers import get_linear_schedule_with_warmup
fem2_analysis[['PTitle','Embarked','Survived']].groupby(['PTitle','Embarked'] ).count()<groupby>
class config:
MAX_LEN = 128
TRAIN_BATCH_SIZE = 64
VALID_BATCH_SIZE = 32
EPOCHS = 3
ELECTRA_PATH = "/kaggle/input/electrabase/electra/base/model"
MODEL_PATH = "pytorch_model.bin"
TRAINING_FILE = ".. /input/tweet-train-folds/train_folds.csv"
TOKENIZER = tokenizers.BertWordPieceTokenizer(
 "/kaggle/input/berthub/assets/vocab.txt",
lowercase=True
)
men_analysis = merged[merged['Sex'] == 'male']
men_analysis[['PTitle','Pclass','Survived','Fsize']].groupby(['PTitle','Pclass','Fsize'] ).mean()<groupby>
PATH="/kaggle/output"
men2_analysis = men_analysis[(men_analysis['Pclass'] == 1)& 
(men_analysis['PTitle'] != 'Master')& 
(men_analysis['Fsize'] != 'Large')]
men2_analysis['HasCabin'] = men2_analysis['Cabin'].notnull()
men2_analysis[['PTitle','HasCabin','Survived']].groupby(['PTitle','HasCabin'] ).mean()<feature_engineering>
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))
merged['Group'] = merged['Sex']
merged.loc[(merged['Sex'] == 'female')& 
(merged['Fsize'] == 'Large')& 
(merged['Pclass'] == 3), 'Group'] = 'Females, large family, class 3'
merged.loc[(merged['PTitle'] == 'Miss')& 
(merged['Fsize'] != 'Large')& 
(merged['Pclass'] == 3)& 
(merged['Embarked'] == 'S'), 'Group'] = 'Miss, class 3, embarked Southampton'
merged.loc[(merged['PTitle'] == 'Master')& 
(( merged['Pclass'] < 3)| 
(merged['Fsize'] == 'Small')) , 'Group'] = 'Masters in Pclass 1&2 or small families'
merged.loc[(merged['PTitle'] == 'Rare')& 
(merged['Sex'] == 'male')& 
(merged['Pclass'] == 1)& 
(merged['Cabin'].notnull())& 
(merged['Fsize'] != 'Large'), 'Group'] = 'Rare title, Cabin Pclass 1, not large'
merged[['Group','Survived']].groupby(['Group'], sort=False ).mean().plot.bar()
plt.show()
merged[['Group','Survived']].groupby(['Group'], sort=False ).mean()<feature_engineering>
class AverageMeter:

 def __init__(self):
self.reset()
def reset(self):
self.val = 0
self.avg = 0
self.sum = 0
self.count = 0
def update(self, val, n=1):
self.val = val
self.sum += val * n
self.count += n
self.avg = self.sum / self.count
class EarlyStopping:
def __init__(self, patience=20, mode="max", delta=0.001):
self.patience = patience
self.counter = 0
self.mode = mode
self.best_score = None
self.early_stop = False
self.delta = delta
if self.mode == "min":
self.val_score = np.Inf
else:
self.val_score = -np.Inf
def __call__(self, epoch_score, model, model_path):
if self.mode == "min":
score = -1.0 * epoch_score
else:
score = np.copy(epoch_score)
 if self.best_score is None:
self.best_score = score
self.save_checkpoint(epoch_score, model, model_path)
 elif score < self.best_score + self.delta:
self.counter += 1
print('EarlyStopping counter: {} out of {}'.format(self.counter, self.patience))
if self.counter >= self.patience:
self.early_stop = True
else:
self.best_score = score
self.save_checkpoint(epoch_score, model, model_path)
 self.counter = 0
def save_checkpoint(self, epoch_score, model, model_path):
if epoch_score not in [-np.inf, np.inf, -np.nan, np.nan]:
print('Validation score improved({} --> {} ).Saving model!'.format(self.val_score, epoch_score))
torch.save(model.state_dict() , model_path)
 self.val_score = epoch_score
merged['Predict'] = 1
merged.loc[merged['Group'] == 'male', 'Predict'] = 0
merged.loc[merged['Group'] == 'Females, large family, class 3', 'Predict'] = 0
merged.loc[merged['Group'] == 'Miss, class 3, embarked Southampton', 'Predict'] = 0
merged.loc[merged['Group'] == 'Masters in Pclass 1&2 or small families', 'Predict'] = 1
merged.loc[merged['Group'] == 'Rare title, Cabin Pclass 1, not large', 'Predict'] = 1
train_set = merged[merged['Survived'].notnull() ]
train_set['Score'] = train_set['Survived'] == train_set['Predict']
print("Accuracy training set:")
 print(train_set['Score'].mean() )<save_to_csv>
def process_data(tweet, selected_text, sentiment, tokenizer, max_len):
len_st = len(selected_text)
 idx0 = None
idx1 = None
for ind in(i for i, e in enumerate(tweet)if e == selected_text[0]):
if tweet[ind: ind+len_st] == selected_text:
idx0 = ind
idx1 = ind + len_st - 1
break
char_targets = [0] * len(tweet)
 if idx0 != None and idx1 != None:
for ct in range(idx0, idx1 + 1):
char_targets[ct] = 1
tok_tweet = tokenizer.encode(tweet)
 input_ids_orig = tok_tweet.ids[1:-1]
tweet_offsets = tok_tweet.offsets[1:-1]
target_idx = []
for j,(offset1, offset2)in enumerate(tweet_offsets):
if sum(char_targets[offset1: offset2])> 0:
target_idx.append(j)
 targets_start = target_idx[0]
targets_end = target_idx[-1]
sentiment_id = {
'positive': 3893,
'negative': 4997,
'neutral': 8699
}
input_ids = [101] + [sentiment_id[sentiment]] + [102] + input_ids_orig + [102]
token_type_ids = [0, 0, 0] + [1] *(len(input_ids_orig)+ 1)
 mask = [1] * len(token_type_ids)
 tweet_offsets = [(0, 0)] * 3 + tweet_offsets + [(0, 0)]
targets_start += 3
targets_end += 3
padding_length = max_len - len(input_ids)
 if padding_length > 0:
input_ids = input_ids +([0] * padding_length)
 mask = mask +([0] * padding_length)
 token_type_ids = token_type_ids +([0] * padding_length)
 tweet_offsets = tweet_offsets +([(0, 0)] * padding_length)
 return {
'ids': input_ids,
'mask': mask,
'token_type_ids': token_type_ids,
'targets_start': targets_start,
'targets_end': targets_end,
'orig_tweet': tweet,
'orig_selected': selected_text,
'sentiment': sentiment,
'offsets': tweet_offsets
}
my_solution = pd.DataFrame({'PassengerId': test_analysed['PassengerId'],
'Survived':test_analysed['Predict']})
 my_solution.to_csv('submission.csv', index = False )<set_options>
class TweetDataset:
def __init__(self, tweet, sentiment, selected_text):
self.tweet = tweet
self.sentiment = sentiment
self.selected_text = selected_text
self.tokenizer = config.TOKENIZER
self.max_len = config.MAX_LEN
def __len__(self):
return len(self.tweet)
 def __getitem__(self, item):
data = process_data(
 self.tweet[item],
self.selected_text[item],
self.sentiment[item],
self.tokenizer,
self.max_len 
)
 return {
'ids': torch.tensor(data["ids"], dtype=torch.long),
'mask': torch.tensor(data["mask"], dtype=torch.long),
'token_type_ids': torch.tensor(data["token_type_ids"], dtype=torch.long),
'targets_start': torch.tensor(data["targets_start"], dtype=torch.long),
'targets_end': torch.tensor(data["targets_end"], dtype=torch.long),
'orig_tweet': data["orig_tweet"],
'orig_selected': data["orig_selected"],
'sentiment': data["sentiment"],
'offsets': torch.tensor(data["offsets"], dtype=torch.long)
 }
%matplotlib inline
py.init_notebook_mode(connected = False)
<load_from_csv>
class TweetModel(transformers.BertPreTrainedModel):
def __init__(self, conf):
super(TweetModel, self ).__init__(conf)
 self.bert = transformers.ElectraModel.from_pretrained(config.ELECTRA_PATH, config=conf)
 self.drop_out = nn.Dropout(0.2)
 self.l0 = nn.Linear(768 * 2, 2)
 torch.nn.init.normal_(self.l0.weight, std=0.002)
 def forward(self, ids, mask, token_type_ids):
_,out = self.bert(
 ids,
attention_mask=mask,
token_type_ids=token_type_ids 
)
 out = torch.cat(( out[-1], out[-2]), dim=-1)
 out = self.drop_out(out)
 logits = self.l0(out)
 start_logits, end_logits = logits.split(1, dim=-1)
 start_logits = start_logits.squeeze(-1)
 end_logits = end_logits.squeeze(-1)
 return start_logits, end_logits
traindf = pd.read_csv('.. /input/train.csv')
 testdf = pd.read_csv('.. /input/test.csv')
 print('The size of traind data is:{}'.format(traindf.shape),'
','The size of test data is:{}'.format(testdf.shape))
traindf.head()<count_missing_values>
def loss_fn(start_logits, end_logits, start_positions, end_positions):
loss_fct = nn.CrossEntropyLoss()
start_loss = loss_fct(start_logits, start_positions)
 end_loss = loss_fct(end_logits, end_positions)
 total_loss =(start_loss + end_loss)
 return total_loss
def train_fn(data_loader, model, optimizer, device, scheduler=None):
model.train()
losses = AverageMeter()
jaccards = AverageMeter()
tk0 = tqdm(data_loader, total=len(data_loader))
for bi, d in enumerate(tk0):
ids = d["ids"]
token_type_ids = d["token_type_ids"]
mask = d["mask"]
targets_start = d["targets_start"]
targets_end = d["targets_end"]
sentiment = d["sentiment"]
orig_selected = d["orig_selected"]
orig_tweet = d["orig_tweet"]
targets_start = d["targets_start"]
targets_end = d["targets_end"]
offsets = d["offsets"]
ids = ids.to(device, dtype=torch.long)
 token_type_ids = token_type_ids.to(device, dtype=torch.long)
 mask = mask.to(device, dtype=torch.long)
 targets_start = targets_start.to(device, dtype=torch.long)
 targets_end = targets_end.to(device, dtype=torch.long)
 model.zero_grad()
outputs_start, outputs_end = model(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids, 
)
 loss = loss_fn(outputs_start, outputs_end, targets_start, targets_end)
 loss.backward()
optimizer.step()
scheduler.step()
outputs_start = torch.softmax(outputs_start, dim=1 ).cpu().detach().numpy()
outputs_end = torch.softmax(outputs_end, dim=1 ).cpu().detach().numpy()
jaccard_scores = []
for px, tweet in enumerate(orig_tweet):
selected_tweet = orig_selected[px]
tweet_sentiment = sentiment[px]
jaccard_score, _ = calculate_jaccard_score(
 original_tweet=tweet,
target_string=selected_tweet,
sentiment_val=tweet_sentiment,
idx_start=np.argmax(outputs_start[px, :]),
idx_end=np.argmax(outputs_end[px, :]),
offsets=offsets[px] 
)
 jaccard_scores.append(jaccard_score)
 jaccards.update(np.mean(jaccard_scores), ids.size(0))
losses.update(loss.item() , ids.size(0))
tk0.set_postfix(loss=losses.avg, jaccard=jaccards.avg)
nulls = traindf.isnull().sum()
nulls[nulls > 1000]<drop_column>
def calculate_jaccard_score(
 original_tweet,
target_string,
sentiment_val,
idx_start,
idx_end,
offsets,
verbose=False):
if idx_end < idx_start:
idx_end = idx_start
filtered_output = ""
for ix in range(idx_start, idx_end + 1):
filtered_output += original_tweet[offsets[ix][0]: offsets[ix][1]]
if(ix+1)< len(offsets)and offsets[ix][1] < offsets[ix+1][0]:
filtered_output += " "
if sentiment_val == "neutral" or len(original_tweet.split())< 2:
filtered_output = original_tweet
jac = jaccard(target_string.strip() , filtered_output.strip())
 return jac, filtered_output
def eval_fn(data_loader, model, device):
model.eval()
losses = AverageMeter()
jaccards = AverageMeter()
with torch.no_grad() :
tk0 = tqdm(data_loader, total=len(data_loader))
for bi, d in enumerate(tk0):
ids = d["ids"]
token_type_ids = d["token_type_ids"]
mask = d["mask"]
sentiment = d["sentiment"]
orig_selected = d["orig_selected"]
orig_tweet = d["orig_tweet"]
targets_start = d["targets_start"]
targets_end = d["targets_end"]
offsets = d["offsets"].numpy()
ids = ids.to(device, dtype=torch.long)
 token_type_ids = token_type_ids.to(device, dtype=torch.long)
 mask = mask.to(device, dtype=torch.long)
 targets_start = targets_start.to(device, dtype=torch.long)
 targets_end = targets_end.to(device, dtype=torch.long)
 outputs_start, outputs_end = model(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids 
)
 loss = loss_fn(outputs_start, outputs_end, targets_start, targets_end)
 outputs_start = torch.softmax(outputs_start, dim=1 ).cpu().detach().numpy()
outputs_end = torch.softmax(outputs_end, dim=1 ).cpu().detach().numpy()
jaccard_scores = []
for px, tweet in enumerate(orig_tweet):
selected_tweet = orig_selected[px]
tweet_sentiment = sentiment[px]
jaccard_score, _ = calculate_jaccard_score(
 original_tweet=tweet,
target_string=selected_tweet,
sentiment_val=tweet_sentiment,
idx_start=np.argmax(outputs_start[px, :]),
idx_end=np.argmax(outputs_end[px, :]),
offsets=offsets[px] 
)
 jaccard_scores.append(jaccard_score)
 jaccards.update(np.mean(jaccard_scores), ids.size(0))
losses.update(loss.item() , ids.size(0))
tk0.set_postfix(loss=losses.avg, jaccard=jaccards.avg)
 print(f"Jaccard = {jaccards.avg}")
 return jaccards.avg
traindf.drop(traindf.loc[(traindf['GrLivArea'] > 4500)&(traindf['SalePrice'] < 300000)].index, axis = 0, inplace = True)
 traindf.drop(traindf.loc[(traindf['LotArea'] > 150000)&(traindf['SalePrice'] < 400000)].index, axis = 0, inplace = True )<compute_test_metric>
def run(fold):
dfx = pd.read_csv(config.TRAINING_FILE)
 df_train = dfx[dfx.kfold != fold].reset_index(drop=True)
 df_valid = dfx[dfx.kfold == fold].reset_index(drop=True)
 train_dataset = TweetDataset(
 tweet=df_train.text.values,
sentiment=df_train.sentiment.values,
selected_text=df_train.selected_text.values 
)
 train_data_loader = torch.utils.data.DataLoader(
 train_dataset,
batch_size=config.TRAIN_BATCH_SIZE,
num_workers=4 
)
 valid_dataset = TweetDataset(
 tweet=df_valid.text.values,
sentiment=df_valid.sentiment.values,
selected_text=df_valid.selected_text.values 
)
 valid_data_loader = torch.utils.data.DataLoader(
 valid_dataset,
batch_size=config.VALID_BATCH_SIZE,
num_workers=2 
)
 device = torch.device("cuda")
 model_config = transformers.ElectraConfig.from_pretrained(config.ELECTRA_PATH)
 model_config.output_hidden_states = True
model = TweetModel(conf=model_config)
 model.to(device)
 num_train_steps = int(len(df_train)/ config.TRAIN_BATCH_SIZE * config.EPOCHS)
 param_optimizer = list(model.named_parameters())
 no_decay = ["bias", "LayerNorm.bias", "LayerNorm.weight"]
optimizer_parameters = [
{'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.001},
{'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0},
]
optimizer = AdamW(optimizer_parameters, lr=3e-5)
 scheduler = get_linear_schedule_with_warmup(
 optimizer,
num_warmup_steps=0,
num_training_steps=num_train_steps 
)
 es = EarlyStopping(patience=2, mode="max")
 print(f"Training is Starting for fold={fold}")
 for epoch in range(config.EPOCHS):
train_fn(train_data_loader, model, optimizer, device, scheduler=scheduler)
 jaccard = eval_fn(valid_data_loader, model, device)
 print(f"Jaccard Score = {jaccard}")
 es(jaccard, model, model_path=f"example_run_{fold}.bin")
 if es.early_stop:
print("Early stopping")
 break
print('The above Histogram has skew: {} and kurtosis: {}'.format(skew(traindf['SalePrice']), kurtosis(traindf['SalePrice'])) )<concatenate>
df_test = pd.read_csv(".. /input/tweet-sentiment-extraction/test.csv")
 df_test.loc[:, "selected_text"] = df_test.text.values
def fulldf(data1, data2):
fulldf = pd.concat([data1, data2])
 return fulldf
def sepdf(data):
data1 = fulldf.iloc[:len(traindf)]
data2 = fulldf.iloc[len(traindf):]
return data1, data2<prepare_x_and_y>
device = torch.device("cuda")
 model_config = transformers.ElectraConfig.from_pretrained(config.ELECTRA_PATH)
 model_config.output_hidden_states = True
target = traindf['SalePrice']
Id = testdf['Id']
fulldf = fulldf(traindf.drop('SalePrice', axis = 1), testdf )<feature_engineering>
model1 = TweetModel(conf=model_config)
 model1.to(device)
 model1.load_state_dict(torch.load(".. /input/tweetmodel1/example_run_5(1 ).bin"))
model1.eval()
features = fulldf.select_dtypes(include = ['object'])
 f = pd.DataFrame()
for fe in features.columns:
l = 0
l = []
l.append(fulldf[fe].unique())
 f[fe] = l
features_dict = {}
for column in f.columns:
new_column = 0
new_column = pd.Series(f[column][0], name = column,)
 keys = 0
keys = range(len(new_column))
for i in keys:
features_dict[column] = new_column
unique_feature_data = pd.DataFrame(features_dict)
 unique_feature_data.fillna(' ', inplace = True)
 pd.set_option('max_columns',None)
 print(' Unique Categorical Features')
 unique_feature_data.head()<sort_values>
final_output = []
test_dataset = TweetDataset(
 tweet=df_test.text.values,
sentiment=df_test.sentiment.values,
selected_text=df_test.selected_text.values 
)
 data_loader = torch.utils.data.DataLoader(
 test_dataset,
shuffle=False,
batch_size=config.VALID_BATCH_SIZE,
num_workers=1 
)
 with torch.no_grad() :
tk0 = tqdm(data_loader, total=len(data_loader))
for bi, d in enumerate(tk0):
ids = d["ids"]
token_type_ids = d["token_type_ids"]
mask = d["mask"]
sentiment = d["sentiment"]
orig_selected = d["orig_selected"]
orig_tweet = d["orig_tweet"]
targets_start = d["targets_start"]
targets_end = d["targets_end"]
offsets = d["offsets"].numpy()
ids = ids.to(device, dtype=torch.long)
 token_type_ids = token_type_ids.to(device, dtype=torch.long)
 mask = mask.to(device, dtype=torch.long)
 targets_start = targets_start.to(device, dtype=torch.long)
 targets_end = targets_end.to(device, dtype=torch.long)
 outputs_start, outputs_end = model1(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids 
)
 outputs_start = torch.softmax(outputs_start, dim=1 ).cpu().detach().numpy()
outputs_end = torch.softmax(outputs_end, dim=1 ).cpu().detach().numpy()
jaccard_scores = []
for px, tweet in enumerate(orig_tweet):
selected_tweet = orig_selected[px]
tweet_sentiment = sentiment[px]
_, output_sentence = calculate_jaccard_score(
 original_tweet=tweet,
target_string=selected_tweet,
sentiment_val=tweet_sentiment,
idx_start=np.argmax(outputs_start[px, :]),
idx_end=np.argmax(outputs_end[px, :]),
offsets=offsets[px] 
)
 final_output.append(output_sentence )
fulldf[['PoolQC','PoolArea',
'GarageQual','GarageArea',
'FireplaceQu','Fireplaces',
'BsmtQual','TotalBsmtSF']].sort_values(by = ['PoolQC','GarageQual','FireplaceQu','BsmtQual','TotalBsmtSF'], na_position = 'first' ).head()<count_missing_values>
def post_process(selected):
return " ".join(set(selected.lower().split()))
<sort_values><EOS> 
sample = pd.read_csv(".. /input/tweet-sentiment-extraction/sample_submission.csv")
 sample.loc[:, 'selected_text'] = final_output
sample.selected_text = sample.selected_text.map(post_process)
 sample.to_csv("submission.csv", index=False )