Kaggle competition: Tweet Sentiment Extraction
<SOS> metric: Jaccard Kaggle data source: tweet-sentiment-extraction<feature_engineering>
import numpy as np
import pandas as pd
import os
import tokenizers
import string
import torch
import transformers
import torch.nn as nn
from torch.nn import functional as F
from tqdm import tqdm
import pickle
import re
import string
for col in skew_cols.index.tolist() :
train_encoded[col] = np.log1p(train_encoded[col])
 test_encoded[col] = test_encoded[col].apply(np.log1p)
 print(test_encoded.dtypes.value_counts())
 print('
\t\t:)Skewed data Transformation Completed :)' )<split>
MAX_LEN = 168
VALID_BATCH_SIZE = 8
EPOCHS = 5
ROBERTA_PATH = ".. /input/roberta-base"
ROBERTA_PATH_NEW = ".. /input/rb-base"
TOKENIZER = tokenizers.ByteLevelBPETokenizer(
 vocab_file=f"{ROBERTA_PATH}/vocab.json",
merges_file=f"{ROBERTA_PATH}/merges.txt",
lowercase=True,
add_prefix_space=True
)
X_train, X_test, y_train, y_test = train_test_split(train_encoded, predictor,
test_size=0.3, random_state=42)
 print(X_train.shape)
 print(y_train.shape)
 print(X_test.shape)
 print(y_test.shape )<compute_test_metric>
class TweetModel(transformers.BertPreTrainedModel):
def __init__(self, conf):
super(TweetModel, self ).__init__(conf)
 self.roberta = transformers.RobertaModel.from_pretrained(ROBERTA_PATH, config=conf)
 self.drop_out = nn.Dropout(0.3)
 self.l0 = nn.Linear(768, 2)
 torch.nn.init.normal_(self.l0.weight, std=0.02)
 def forward(self, ids, mask, token_type_ids):
seq, pooled = self.roberta(
 ids,
attention_mask=mask,
token_type_ids=token_type_ids 
)
 out = self.drop_out(seq)
 logits = self.l0(out)
 start_logits, end_logits = logits.split(1, dim=-1)
 start_logits = start_logits.squeeze(-1)
 end_logits = end_logits.squeeze(-1)
 return start_logits, end_logits
def rmse(true_data, predicted_data):
return np.sqrt(mean_squared_error(true_data, predicted_data))<choose_model_class>
link_re = re.compile('http[s]?://\S+')
 re_username = '^(\_)\w+'
re_username2 = '^@(\_)\w+'
def clean_text(text, sentiment):
cleaned_text = " ".join(str(text ).split() ).strip()
if sentiment != 'neutral':
if '_it_good' in text or '_in_love' in text or '_violence' in text:
return text
if re.search(re_username, cleaned_text):
cleaned_text = re.sub(re_username, '', cleaned_text ).strip()
if re.search(re_username2, cleaned_text):
cleaned_text = re.sub(re_username2, '', cleaned_text ).strip()
if re.search(link_re, cleaned_text):
split_text = text.split()
if re.search(link_re, split_text[0])or re.search(link_re, split_text[-1]):
cleaned_text = re.sub('http[s]?://\S+', '', text ).strip()
if cleaned_text.split() [0] == '-':
cleaned_text = cleaned_text[1:].strip()
if 'these dogs are going to die if somebody doesn`t save them!' in cleaned_text:
cleaned_text = 'these dogs are going to die if somebody doesn`t save them!'
elif cleaned_text == 'imo':
cleaned_text = 'Thanks'
elif cleaned_text == 'and_jay hi! **** your job!':
cleaned_text = 'hi! **** your job!'
elif cleaned_text == 'beckett Thanks':
cleaned_text = 'Thanks'
elif 'c Thank' in cleaned_text:
cleaned_text = 'Thank'
elif cleaned_text == 'hall no i was gutted when he wasn`t.lmao.i think i`m obsessed with him, bahaha.':
cleaned_text = 'no i was gutted when he wasn`t.lmao.i think i`m obsessed with him, bahaha.'
return cleaned_text
linearRegression = LinearRegression().fit(X_train, y_train)
 prediction1 = linearRegression.predict(X_test)
 LR_score = linearRegression.score(X_test, y_test)
 LR_rmse = rmse(y_test, prediction1)
 print('The scoring and root mean squared error for Linear Regression in percentage
')
 print('\t\tThe score is: ',LR_score*100)
 print('\t\tThe rmse is : ',LR_rmse*100 )<define_search_space>
def process_data(tweet, selected_text, sentiment, tokenizer, max_len):
tweet = " " + " ".join(str(tweet ).split())
 selected_text = " " + " ".join(str(selected_text ).split())
 len_st = len(selected_text)- 1
idx0 = None
idx1 = None
for ind in(i for i, e in enumerate(tweet)if e == selected_text[1]):
if " " + tweet[ind: ind+len_st] == selected_text:
idx0 = ind
idx1 = ind + len_st - 1
break
char_targets = [0] * len(tweet)
 if idx0 != None and idx1 != None:
for ct in range(idx0, idx1 + 1):
char_targets[ct] = 1
tok_tweet = tokenizer.encode(tweet)
 input_ids_orig = tok_tweet.ids
tweet_offsets = tok_tweet.offsets
target_idx = []
for j,(offset1, offset2)in enumerate(tweet_offsets):
if sum(char_targets[offset1: offset2])> 0:
target_idx.append(j)
 targets_start = target_idx[0]
targets_end = target_idx[-1]
sentiment_id = {
'positive': [1313],
'negative': [2430],
'neutral': [7974]
}
input_ids = [0] + sentiment_id[sentiment] + [2] + [2] + input_ids_orig + [2]
token_type_ids = [0, 0, 0, 0] + [0] *(len(input_ids_orig)+ 1)
 mask = [1] * len(token_type_ids)
 tweet_offsets = [(0, 0)] * 4 + tweet_offsets + [(0, 0)]
targets_start += 4
targets_end += 4
padding_length = max_len - len(input_ids)
 if padding_length > 0:
input_ids = input_ids +([1] * padding_length)
 mask = mask +([0] * padding_length)
 token_type_ids = token_type_ids +([0] * padding_length)
 tweet_offsets = tweet_offsets +([(0, 0)] * padding_length)
 return {
'ids': input_ids,
'mask': mask,
'token_type_ids': token_type_ids,
'targets_start': targets_start,
'targets_end': targets_end,
'orig_tweet': tweet,
'orig_selected': selected_text,
'sentiment': sentiment,
'offsets': tweet_offsets
}
alphas = [0.005, 0.05, 0.1, 1, 5, 10, 50, 100]<compute_train_metric>
class TweetDataset:
def __init__(self, tweet, sentiment, selected_text):
self.tweet = tweet
self.sentiment = sentiment
self.selected_text = selected_text
self.tokenizer = TOKENIZER
self.max_len = MAX_LEN
def __len__(self):
return len(self.tweet)
 def __getitem__(self, item):
data = process_data(
 self.tweet[item],
self.selected_text[item],
self.sentiment[item],
self.tokenizer,
self.max_len 
)
 return {
'ids': torch.tensor(data["ids"], dtype=torch.long),
'mask': torch.tensor(data["mask"], dtype=torch.long),
'token_type_ids': torch.tensor(data["token_type_ids"], dtype=torch.long),
'targets_start': torch.tensor(data["targets_start"], dtype=torch.long),
'targets_end': torch.tensor(data["targets_end"], dtype=torch.long),
'orig_tweet': data["orig_tweet"],
'orig_selected': data["orig_selected"],
'sentiment': data["sentiment"],
'offsets': torch.tensor(data["offsets"], dtype=torch.long)
 }
ridgeCV = RidgeCV(alphas=alphas ).fit(X_train, y_train)
 prediction2 = ridgeCV.predict(X_test)
 R_score = ridgeCV.score(X_test, y_test)
 R_rmse = rmse(y_test, prediction2)
 print('The scoring and root mean squared error for Linear Regression in percentage
')
 print('\tThe parameter used for here was alpha = {}
'.format(ridgeCV.alpha_))
print('\t\tThe score is: ',R_score*100)
 print('\t\tThe rmse is : ',R_rmse*100 )<compute_train_metric>
punc = "!."
def postprocess(text, predicted_text):
splitted = text.split(predicted_text)[0]
sub = len(splitted)- len(" ".join(splitted.split()))
splitted1 = text.split(predicted_text.strip())[0]
sub1 = len(splitted1)- len(" ".join(splitted1.split()))
if sub1 == 2 and text.strip() != predicted_text.strip() and text[:2] == " ":
predicted_text = predicted_text.strip()
start = text.find(predicted_text)
 end = start + len(predicted_text)
 if start == 0:
end = end - 2
else:
start = start - 2
if predicted_text[-1] in "!.,":
end = end - 1
predicted_text = text[start: end]
elif sub > 1 and text.strip() != predicted_text.strip() :
if len(predicted_text.split())== 1:
if text[0] == " ":
text = text[1:] + " "
else:
predicted_text = predicted_text + " "
if text[0] == " " and text.strip().find(predicted_text.strip())== 0:
text = " " + text[:-1]
add_one = False
if predicted_text[-1] not in punc:
add_one = True
splitted = text.split(predicted_text)[0]
sub = len(splitted)- len(" ".join(splitted.split()))
start = text.find(predicted_text)- sub
if start < 0:
start = text.find(predicted_text.strip())
 add_one = False
predicted_text = predicted_text.strip()
if add_one:
end = start + len(predicted_text)+ 1
else:
end = start + len(predicted_text)
 predicted_text = text[start: end]
return predicted_text
lassoCV = LassoCV(alphas=[0.005, 0.001, 0.05, 0.01,1, 5], max_iter=1e2 ).fit(X_train, y_train)
 prediction3 = lassoCV.predict(X_test)
 L_score = lassoCV.score(X_test, y_test)
 L_rmse = rmse(y_test, prediction3)
 print('The scoring and root mean squared error for Linear Regression in percentage
')
 print('\tThe parameter used for here was alpha = {}'.format(lassoCV.alpha_))
print('
\t\tThe score is: ',L_score*100)
 print('\t\tThe rmse is : ',L_rmse*100 )<compute_train_metric>
l1_ratios = np.linspace(0.1, 0.9, 9)
 elasticnetCV = ElasticNetCV(alphas=alphas, l1_ratio=l1_ratios, max_iter=1e2 ).fit(X_train, y_train)
 prediction4 = elasticnetCV.predict(X_test)
 EN_score = elasticnetCV.score(X_test, y_test)
 EN_rmse = rmse(y_test, prediction4)
 print('The scoring and root mean squared error for Linear Regression in percentage
')
 print('\tThe parameter used for here was alpha = {} and l1_ratios = {}
'.format(elasticnetCV.alpha_, elasticnetCV.l1_ratio_))
print('\t\tThe score is: ',EN_score*100)
 print('\t\tThe rmse is : ',EN_rmse*100 )<choose_model_class>
def calculate_jaccard_score(
 original_tweet,
target_string,
sentiment_val,
idx_start,
idx_end,
offsets,
verbose=False):
if idx_end < idx_start:
idx_end = idx_start
filtered_output = ""
for ix in range(idx_start, idx_end + 1):
filtered_output += original_tweet[offsets[ix][0]: offsets[ix][1]]
if(ix+1)< len(offsets)and offsets[ix][1] < offsets[ix+1][0]:
filtered_output += " "
if len(original_tweet.split())< 2:
filtered_output = original_tweet
if sentiment_val != "neutral" and verbose == True:
if filtered_output.strip().lower() != target_string.strip().lower() :
print("********************************")
 print(f"Output= {filtered_output.strip() }")
 print(f"Target= {target_string.strip() }")
 print(f"Tweet= {original_tweet.strip() }")
 print("********************************")
 jac = 0
return jac, filtered_output
randfr = RandomForestRegressor(random_state = 42 )<define_search_space>
df_test = pd.read_csv(".. /input/tweet-sentiment-extraction/test.csv")
 df_test.loc[:, "selected_text"] = df_test.text.values
param_grid = {
'n_estimators': [50,250,500],
'max_features': ['auto', 'sqrt', 'log2'],
'max_depth' : [2, 4, 6, 8, 10],
}<train_on_grid>
randfr_cv = GridSearchCV(estimator=randfr, param_grid=param_grid, cv= 5)
 randfr = randfr_cv.fit(X_train, y_train )<compute_train_metric>
device = torch.device("cuda")
 model_config = transformers.RobertaConfig.from_pretrained(ROBERTA_PATH)
prediction5 = randfr.predict(X_test)
 RF_score = randfr.score(X_test, y_test)
 RF_rmse = rmse(y_test, prediction5)
 print('The scoring and root mean squared error for Linear Regression in percentage
')
 print('\tThe parameter used for here were = {}
'.format(randfr_cv.best_params_))
print('\t\tThe score for random forest is: {} '.format(RF_score*100))
print('\t\tThe rmse is: {} '.format(RF_rmse*100))<prepare_output>
MODEL_BASE_PATH_OLD = '.. /input/roberta-base-uncased4'
MODEL_BASE_PATH = '.. /input/roberta-pp2'
score_vals = [LR_score, R_score, L_score, EN_score, RF_score]
rmse_vals = [LR_rmse, R_rmse, L_rmse, EN_rmse, RF_rmse]
labels = ['Linear', 'Ridge', 'Lasso', 'ElasticNet', 'RandomForest']
rmse_df = pd.Series(score_vals, index=labels ).to_frame()
rmse_df.rename(columns={0: 'SCORES'}, inplace=1)
 rmse_df['RMSE'] = rmse_vals
rmse_df<sort_values>
ENSEMBLES = [
{'model': TweetModel(conf=model_config), 'state_dict': f"{MODEL_BASE_PATH}/model_0.bin", 'weight': 1},
{'model': TweetModel(conf=model_config), 'state_dict': f"{MODEL_BASE_PATH}/model_1.bin", 'weight': 1},
{'model': TweetModel(conf=model_config), 'state_dict': f"{MODEL_BASE_PATH}/model_2.bin", 'weight': 1},
{'model': TweetModel(conf=model_config), 'state_dict': f"{MODEL_BASE_PATH}/model_3.bin", 'weight': 1},
{'model': TweetModel(conf=model_config), 'state_dict': f"{MODEL_BASE_PATH}/model_4.bin", 'weight': 1},
{'model': TweetModel(conf=model_config), 'state_dict': f"{MODEL_BASE_PATH}/model_5.bin", 'weight': 1},
{'model': TweetModel(conf=model_config), 'state_dict': f"{MODEL_BASE_PATH}/model_6.bin", 'weight': 1},
{'model': TweetModel(conf=model_config), 'state_dict': f"{MODEL_BASE_PATH}/model_7.bin", 'weight': 1},
]
rmse_df = rmse_df.sort_values(['RMSE'], ascending=True)
 rmse_df<train_model>
models = []
weights = []
for val in ENSEMBLES:
model = val['model']
model.to(device)
 model.load_state_dict(torch.load(val['state_dict']))
model.eval()
models.append(model)
 weights.append(val['weight'] )
start_xgb = datetime.now()
xgb = XGBRegressor().fit(X_train, y_train)
 end_xgb = datetime.now()
xgb_time = end_xgb - start_xgb
print('Duration for XGBoost: {}'.format(xgb_time))<compute_test_metric>
def get_best_start_end_idxs(_start_logits, _end_logits):
best_logit = -1000
best_idxs = None
for start_idx, start_logit in enumerate(_start_logits):
for end_idx, end_logit in enumerate(_end_logits[start_idx:]):
logit_sum =(start_logit + end_logit ).item()
if logit_sum > best_logit:
best_logit = logit_sum
best_idxs =(start_idx, start_idx+end_idx)
 return best_idxs
prediction6 = xgb.predict(X_test)
 xgb_score = xgb.score(X_test, y_test)
 xgb_rmse = rmse(y_test, prediction6)
 print('The scoring and root mean squared error for XGBoost in percentage
')
 print('\t\tThe score is: ',xgb_score*100)
 print('\t\tThe rmse is : ',xgb_rmse*100 )<train_model>
final_output = []
test_dataset = TweetDataset(
 tweet=df_test.text.values,
sentiment=df_test.sentiment.values,
selected_text=df_test.selected_text.values 
)
 data_loader = torch.utils.data.DataLoader(
 test_dataset,
shuffle=False,
batch_size=VALID_BATCH_SIZE,
num_workers=1 
)
 with torch.no_grad() :
tk0 = tqdm(data_loader, total=len(data_loader))
for bi, d in enumerate(tk0):
ids = d["ids"]
token_type_ids = d["token_type_ids"]
mask = d["mask"]
sentiment = d["sentiment"]
orig_selected = d["orig_selected"]
orig_tweet = d["orig_tweet"]
targets_start = d["targets_start"]
targets_end = d["targets_end"]
offsets = d["offsets"].numpy()
ids = ids.to(device, dtype=torch.long)
 token_type_ids = token_type_ids.to(device, dtype=torch.long)
 mask = mask.to(device, dtype=torch.long)
 targets_start = targets_start.to(device, dtype=torch.long)
 targets_end = targets_end.to(device, dtype=torch.long)
 outputs_start = []
outputs_end = []
for index, model in enumerate(models):
output_start, output_end = model(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids 
)
 outputs_start.append(output_start * weights[index])
 outputs_end.append(output_end * weights[index])
 outputs_start = sum(outputs_start)/ len(outputs_start)
 outputs_end = sum(outputs_end)/ len(outputs_end)
 outputs_start = torch.softmax(outputs_start, dim=1 ).cpu().detach().numpy()
outputs_end = torch.softmax(outputs_end, dim=1 ).cpu().detach().numpy()
jaccard_scores = []
for px, tweet in enumerate(orig_tweet):
selected_tweet = orig_selected[px]
tweet_sentiment = sentiment[px]
idx_start, idx_end = get_best_start_end_idxs(outputs_start[px, :], outputs_end[px, :])
 _, output_sentence = calculate_jaccard_score(
 original_tweet=tweet,
target_string=selected_tweet,
sentiment_val=tweet_sentiment,
idx_start=idx_start,
idx_end=idx_end,
offsets=offsets[px] 
)
 final_output.append(output_sentence )
start_lgbm = datetime.now()
lgb = LGBMRegressor().fit(X_train, y_train)
 end_lgbm = datetime.now()
lgbm_time = end_lgbm - start_lgbm
print('Duration for Light GBM: {}'.format(lgbm_time))<compute_test_metric>
df_test['selected_text'] = final_output
df_test['selected_text'] = df_test.apply(lambda x: postprocess(x.text, x.selected_text), axis=1 )
prediction7 = lgb.predict(X_test)
 lgb_score = lgb.score(X_test, y_test)
 lgb_rmse = rmse(y_test, prediction7)
 print('The scoring and root mean squared error for light GBM in percentage
')
 print('\t\tThe score is: ',lgb_score*100)
 print('\t\tThe rmse is : ',lgb_rmse*100 )<create_dataframe>
def place_in_back(x):
splitted = x.text.split(x.selected_text)[0]
sub = len(splitted)- len(" ".join(splitted.split()))
select=x.selected_text
ind = x.text.find(x.selected_text.strip())-1
if(( ind >0)&(select.startswith('.')!=1)&(sub>0)) :
if(( x.text[ind] in string.punctuation)&(x.sentiment!='neutral')&(sub==1)) :
select = x.text[ind]+select
print(select,sub)
 elif(( x.text[ind-1] in string.punctuation)&(x.sentiment!='neutral')&(sub==2)) :
select = x.text[ind-1]+""+select
print(select,sub)
 return select
comparisons = {'Scores':(lgb_score, xgb_score), 'RMSE':(lgb_rmse, xgb_rmse), 'Execution Time' :(lgbm_time, xgb_time)}
comparisons_df = pd.DataFrame(comparisons )<rename_columns>
df_test['selected_text']=df_test.apply(lambda x : place_in_back(x),axis=1 )
<count_missing_values><EOS> 
sample = pd.read_csv(".. /input/tweet-sentiment-extraction/sample_submission.csv")
 sample.loc[:, 'selected_text'] = df_test['selected_text']
sample['selected_text'] = sample['selected_text'].apply(lambda x: x.replace('!!!', '!')if len(x.split())==1 else x)
 sample['selected_text'] = sample['selected_text'].apply(lambda x: x.replace('!!!!', '!!')if len(x.split())==1 else x)
 sample['selected_text'] = sample['selected_text'].apply(lambda x: x.replace('.. ', '.')if len(x.split())==1 else x)
 sample['selected_text'] = sample['selected_text'].apply(lambda x: x.replace('...', '.')if len(x.split())==1 else x)
 sample['selected_text'] = sample['selected_text'].apply(lambda x: x.replace('.... ', '.. ')if len(x.split())==1 else x)
 sample.to_csv("submission.csv", index=False)