Kaggle competition: Tweet Sentiment Extraction
<SOS> metric: Jaccard Kaggle data source: tweet-sentiment-extraction<define_variables>
tqdm.pandas()
BATCH_SIZE = 1
LEARNING_RATE = 0.001
EPOCHS = 800
INPUT_NODES = 8<prepare_x_and_y>
train_dataset = pd.read_csv('/kaggle/input/tweet-sentiment-extraction/train.csv' )
X_train_torch = torch.from_numpy(X_train.values ).type(torch.FloatTensor)
 y_train_torch = torch.from_numpy(y_train.values ).type(torch.LongTensor)
 X_val_torch = torch.from_numpy(X_val.values ).type(torch.FloatTensor)
 y_val_torch = torch.from_numpy(y_val.values ).type(torch.LongTensor)
 x_test_torch = torch.from_numpy(x_test.values ).type(torch.FloatTensor)
 train = torch.utils.data.TensorDataset(X_train_torch,y_train_torch)
 val = torch.utils.data.TensorDataset(X_val_torch, y_val_torch)
 test = torch.utils.data.TensorDataset(x_test_torch)
 data_loader = torch.utils.data.DataLoader(train)
 val_loader = torch.utils.data.DataLoader(val)
 test_loader = torch.utils.data.DataLoader(test )<set_options>
train_dataset.isnull().sum()
%%markdown
Input Features --> Fully Connected layer(512 nodes)--> Dropout(50%)--> Fully Connected layer(256 nodes)--> Dropout(50%)--> Fully Connected layer(128 nodes)--> Dropout(50%)--> Fully Connected layer(1 node )<choose_model_class>
train_dataset.dropna(inplace=True )
class Titanic_NN(nn.Module):

 def __init__(self, INPUT_NODES):
super(Titanic_NN, self ).__init__()
self.fc1 = nn.Linear(INPUT_NODES,512)
 self.fc2 = nn.Linear(512,256)
 self.dropout = nn.Dropout(0.5)
 self.fc3 = nn.Linear(256, 128)
 self.fc4 = nn.Linear(128,1)
 def forward(self, x):
x = self.fc1(x)
 x = F.relu(x)
 x = self.dropout(x)
 x = F.relu(self.fc2(x))
x = self.dropout(x)
 x = self.fc3(x)
 x = F.relu(x)
 x = self.dropout(x)
 x = self.fc4(x)
 return F.sigmoid(x )<choose_model_class>
def remove_leading_truncuated_word(row):
text = row['text']
selected_text = row['selected_text']
first_selected_text_word = selected_text.lower().split() [0]
if(first_selected_text_word not in text.lower().split())and \ 
(len(first_selected_text_word)== 1):
return " ".join(selected_text.lower().split() [1:])
 else:
return selected_text
def modified_text_to_bert_input(text, tokenizer):
return tokenizer.convert_tokens_to_string(tokenizer.tokenize(text))
model = Titanic_NN(INPUT_NODES)
 try:
model.load_state_dict(torch.load(input_path+'titanic_model_4layers'))
except:
pass
error = nn.BCELoss()
optimizer = torch.optim.SGD(model.parameters() , lr=LEARNING_RATE)
 print(model )<train_model>
tokenizer = BertTokenizer.from_pretrained('/kaggle/input/bertbaseuncased', max_len=192)
 train_dataset['text'] = train_dataset['text'].apply(modified_text_to_bert_input, args=(tokenizer,))
train_dataset['selected_text'] = train_dataset['selected_text'].apply(modified_text_to_bert_input, args=(tokenizer,))
def fit(model, data, phase='training', batch_size = 1, is_cuda=False, input_dim = 8):

 if phase == 'training':
model.train()
elif phase == 'validation':
model.eval()
loss_values = 0.0
correct_values = 0
for _,(features, label)in enumerate(data):
if is_cuda:
features, label = features.cuda() , label.cuda()
features, label = Variable(features.view(batch_size, 1, input_dim)) , Variable(label.float().view(-1, 1))
if phase == 'training':
optimizer.zero_grad()
output = model(features)
 loss = error(output, label)
 loss_values += loss.data
if output[0] > 0.5:
predictions = torch.Tensor([1])
 else:
predictions = torch.Tensor([0])
 correct_values += predictions.eq(label.data.view_as(predictions)).cpu().sum()
if phase == 'training':
loss.backward()
optimizer.step()
loss = loss_values / len(data.dataset)
 accuracy = 100.* correct_values / len(data.dataset)
 {phase} loss is {loss:{5}.{2}} and {phase} accuracy is \
=============================================')
 return loss, accuracy<train_model>
train_dataset['selected_text'] = train_dataset.apply(remove_leading_truncuated_word, axis=1)
 train_dataset['selected_text'].replace('', np.nan, inplace=True)
 train_dataset.dropna(inplace=True )
train_loss_list, val_loss_list = [], []
train_accuracy_list, val_accuracy_list = [], []
for epoch in range(EPOCHS):
train_epoch_loss, train_epoch_accuracy = fit(model, data_loader, batch_size=BATCH_SIZE, input_dim=INPUT_NODES)
 val_epoch_loss, val_epoch_accuracy = fit(model, val_loader, phase='validation', batch_size=BATCH_SIZE, input_dim=INPUT_NODES)
 if epoch % 50 == 0:
torch.save(model.state_dict() , 'titanic_model_4layers')
 train_loss_list.append(train_epoch_loss)
 train_accuracy_list.append(train_epoch_accuracy)
 val_loss_list.append(val_epoch_loss)
 val_accuracy_list.append(val_epoch_accuracy )<concatenate>
def is_substring(row):
return row.loc['selected_text'] in row.loc['text']
train_dataset['is_substring'] = train_dataset.apply(is_substring, axis=1)
 train_dataset['is_substring'].value_counts()
accuracies_list.append(accuracies('Neural Network __Validation_Set__', val_accuracy_list[-1]))<predict_on_test>
neutral_dataset = train_dataset[train_dataset['sentiment'] == 'neutral']
neutral_dataset['is_same'] = neutral_dataset['selected_text'] == neutral_dataset['text']
neutral_dataset['is_same'].value_counts()
def predict(model, data):

 model.eval()
test_predictions = list()
for _,(feature,)in enumerate(data):
feature = Variable(feature.view(1, 1, INPUT_NODES))
output = model(feature)
 if output[0] > 0.5:
prediction = 1
else:
prediction = 0
test_predictions.append(prediction)
 return test_predictions<save_to_csv>
train_dataset = train_dataset[~(train_dataset['sentiment'] == 'neutral')]
pred_df = pd.DataFrame(np.c_[np.arange(892, len(test_set)+892)[:,None], predict(model, test_loader)],
columns=['PassengerId', 'Survived'])
 pred_df.to_csv('titanic_submission.csv', index=False )<load_from_csv>
def get_start_and_end_char_index(row):
text = row['text']
selected_text = row['selected_text']
start_idx = text.find(selected_text)
 end_idx = start_idx + len(selected_text)
 return start_idx, end_idx
def target_answer(row, tokenizer):
def is_whitespace(c):
if c == " " or c == "\t" or c == "\r" or c == "
" or ord(c)== 0x202F:
return True
return False
doc_tokens = []
char_to_word_offset = []
prev_is_whitespace = True
for c in row['text']:
if is_whitespace(c):
prev_is_whitespace = True
else:
if prev_is_whitespace:
doc_tokens.append(c)
 else:
doc_tokens[-1] += c
prev_is_whitespace = False
char_to_word_offset.append(len(doc_tokens)- 1)
 start_idx = row['start_char_idx']
end_idx = row['end_char_idx']
start_position = char_to_word_offset[start_idx]
end_position = char_to_word_offset[end_idx - 1]
actual_text = " ".join(doc_tokens[start_position:end_position+1])
 tok_to_orig_index = []
orig_to_tok_index = []
all_doc_tokens = []
for i, token in enumerate(doc_tokens):
orig_to_tok_index.append(len(all_doc_tokens))
sub_tokens = tokenizer.tokenize(token)
 for sub_token in sub_tokens:
tok_to_orig_index.append(i)
 all_doc_tokens.append(sub_token)
 tok_start_position = orig_to_tok_index[start_position]
if end_position < len(doc_tokens)- 1:
tok_end_position = orig_to_tok_index[end_position + 1] - 1
else:
tok_end_position = len(all_doc_tokens)- 1
tok_answer_text = tokenizer.tokenize(row['selected_text'])
 for new_start in range(tok_start_position, tok_end_position + 1):
for new_end in range(tok_end_position, tok_start_position - 1, -1):
text_span = all_doc_tokens[new_start : new_end + 1]
if text_span == tok_answer_text:
return new_start, new_end, tokenizer.convert_tokens_to_string(text_span)
 text_span = all_doc_tokens[tok_start_position : tok_end_position + 1]
return tok_start_position, tok_end_position, tokenizer.convert_tokens_to_string(text_span )
data_test = pd.read_csv('.. /input/titanic/test.csv',index_col='PassengerId')
 data_train = pd.read_csv('.. /input/titanic/train.csv',index_col='PassengerId')
 data_train<count_missing_values>
train_dataset[['start_char_idx', 'end_char_idx']] = train_dataset.apply(get_start_and_end_char_index, axis=1, result_type='expand' )
data_train.isnull().sum() 
<count_unique_values>
train_dataset.isnull().sum()
for i in data_train.columns:
print(i ,': ',len(data_train[i].unique())) 
<drop_column>
train_dataset['is_same'] = train_dataset['selected_text'] == train_dataset['target_answer']
train_dataset['is_same'].value_counts()
columnsForDrop = ['Name', 'Cabin','Ticket','SibSp','Parch']
data_train.drop(columns=columnsForDrop, inplace=True)
 data_test.drop(columns=columnsForDrop, inplace=True)
 data_train<count_values>
class TweetDataset(Dataset):
def __init__(self, df, tokenizer):
self.text = df['text'].tolist()
self.selected_text = df['selected_text'].tolist()
self.target_start_idx = df['target_start_idx'].tolist()
self.target_end_idx = df['target_end_idx'].tolist()
self.sentiment = df['sentiment'].tolist()
self.tokenizer = tokenizer
def __len__(self):
return len(self.text)
 def __getitem__(self, idx):
encoded = self._process_data(self.text[idx], self.sentiment[idx], self.target_start_idx[idx], self.target_end_idx[idx])
 data = {
'input_ids': torch.tensor(encoded['input_ids'], dtype=torch.long),
'token_type_ids': torch.tensor(encoded['token_type_ids'], dtype=torch.long),
'attention_mask': torch.tensor(encoded['attention_mask'], dtype=torch.float),
'target_start_idx': torch.tensor(encoded['target_start_idx'], dtype=torch.long),
'target_end_idx': torch.tensor(encoded['target_end_idx'], dtype=torch.long)
 }
return data
def _process_data(self, text, sentiment, target_start_idx, target_end_idx):
data = self.tokenizer.encode_plus(sentiment, text, pad_to_max_length=True)
 data['target_start_idx'] = target_start_idx + 3
data['target_end_idx'] = target_end_idx + 3
return data
print(data_train.Sex.value_counts())
 print('----------------------------------------------')
 print(data_train.Embarked.value_counts() )<prepare_x_and_y>
class SentimentExtractionBert(nn.Module):
def __init__(self):
super(SentimentExtractionBert, self ).__init__()
self.num_labels = 2
self.bert = BertModel.from_pretrained('/kaggle/input/bertbaseuncased/')
 self.dropout = nn.Dropout(0.2)
 self.outputs = nn.Linear(768 * 2, self.num_labels)
 nn.init.zeros_(self.outputs.bias)
 nn.init.normal_(self.outputs.weight, mean=0.0, std=0.02)
 def forward(self,
input_ids,
attention_mask,
token_type_ids, 
):
outputs = self.bert(
 input_ids=input_ids,
attention_mask=attention_mask,
token_type_ids=token_type_ids 
)
 sequence_output = outputs[0]
sentiment_rep = sequence_output[:,1].unsqueeze(1 ).repeat(1, 192, 1)
 sent_seq_output = torch.cat([sequence_output, sentiment_rep], dim=-1)
 logits = self.outputs(self.dropout(sent_seq_output))
start_logits, end_logits = logits.split(1, dim=-1)
 start_logits = start_logits.squeeze(-1)
 end_logits = end_logits.squeeze(-1)
 return start_logits, end_logits
y = data_train.Survived
X = data_train.drop(columns=['Survived'] )<split>
FOLD = 5
EPOCHS = 3
BATCH_SIZE = 32
LEARNING_RATE = 3e-5
WEIGHT_DECAY = 0.01
WARMUP_STEPS = 0
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu' )
X_train, X_test, y_train, y_test = train_test_split(X, y)
<categorify>
kf = KFold(n_splits=FOLD)
 kfold_list = list(kf.split(train_dataset))
my_imputer = SimpleImputer()
imputed_X_train = pd.DataFrame(my_imputer.fit_transform(X_train))
imputed_X_test = pd.DataFrame(my_imputer.transform(X_test))
imputed_X_train.columns = X_train.columns
imputed_X_test.columns = X_test.columns 
<import_modules>
def train_one_epoch(model, train_dl, optimizer, scheduler):
criterion = nn.CrossEntropyLoss()
training_loss = 0
model.train()
for data in tqdm(train_dl):
input_ids = data['input_ids'].to(device)
 token_type_ids = data['token_type_ids'].to(device)
 attention_mask = data['attention_mask'].to(device)
 target_start_idx = data['target_start_idx'].to(device)
 target_end_idx = data['target_end_idx'].to(device)
 optimizer.zero_grad()
start_logits, end_logits = model(input_ids, attention_mask, token_type_ids)
 loss = criterion(start_logits, target_start_idx)+ criterion(end_logits, target_end_idx)
 loss.backward()
optimizer.step()
scheduler.step()
training_loss += loss.item()
training_loss /= len(train_dl)
 return training_loss
from sklearn.tree import DecisionTreeClassifier
from sklearn.metrics import accuracy_score, classification_report, f1_score
from sklearn.neighbors import KNeighborsClassifier 
<train_on_grid>
def train(fold):
model = SentimentExtractionBert()
model = model.to(device)
 train_ds = TweetDataset(train_dataset.iloc[kfold_list[fold][0]], tokenizer)
 train_dl = DataLoader(train_ds, batch_size=BATCH_SIZE, shuffle=True, drop_last=True)
 no_decay = ["bias", "LayerNorm.weight"]
optimizer_grouped_parameters = [
{
"params": [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)],
"weight_decay": WEIGHT_DECAY,
},
{ "params": [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)],
"weight_decay": 0.0
},
]
optimizer = AdamW(optimizer_grouped_parameters, lr=LEARNING_RATE, betas=(0.9, 0.98), eps=1e-8)
 t_total = len(train_dl)* EPOCHS
scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=WARMUP_STEPS, num_training_steps=t_total)
 for epoch in range(EPOCHS):
training_loss = train_one_epoch(model, train_dl, optimizer, scheduler)
 with open("./log.txt", 'a')as f:
f.write("Fold {}, Epoch {}, Learning Rate: {},Loss: {}
".format(fold, epoch, optimizer.param_groups[0]['lr'], training_loss))
torch.save(model.state_dict() , 'ft_ckpt_{}.pth'.format(fold))
parameters = {'max_depth': list(range(6, 30, 10)) ,
'max_leaf_nodes': list(range(50, 500, 100)) ,
'n_estimators': list(range(50, 1001, 150)) }
gsearch = GridSearchCV(estimator=RandomForestClassifier() ,
param_grid = parameters,
scoring='f1',
n_jobs=4,cv=5,verbose=7)
 gsearch.fit(imputed_X_train, y_train )<find_best_params>
for fold in range(FOLD):
train(fold )
print(gsearch.best_params_.get('max_leaf_nodes'))
print(gsearch.best_params_.get('max_depth'))<count_missing_values>
model0 = SentimentExtractionBert()
model0.load_state_dict(torch.load("ft_ckpt_0.pth"))
model0.to(device)
 model0.eval()
model1 = SentimentExtractionBert()
model1.load_state_dict(torch.load("ft_ckpt_1.pth"))
model1.to(device)
 model1.eval()
model2 = SentimentExtractionBert()
model2.load_state_dict(torch.load("ft_ckpt_2.pth"))
model2.to(device)
 model2.eval()
model3 = SentimentExtractionBert()
model3.load_state_dict(torch.load("ft_ckpt_3.pth"))
model3.to(device)
 model3.eval()
model4 = SentimentExtractionBert()
model4.load_state_dict(torch.load("ft_ckpt_4.pth"))
model4.to(device)
 model4.eval()
data_test.Age.fillna(X.Age.mean() , inplace=True)
 data_test.Fare.fillna(X.Fare.mean() , inplace=True)
 data_test.isna().sum()<predict_on_test>
def process_test_data(row):
data = tokenizer.encode_plus(row['sentiment'], row['text'], pad_to_max_length=True)
 tensor_data = {
'input_ids': torch.tensor(data['input_ids'], dtype=torch.long ).unsqueeze(0),
'token_type_ids': torch.tensor(data['token_type_ids'], dtype=torch.long ).unsqueeze(0),
'attention_mask': torch.tensor(data['attention_mask'], dtype=torch.float ).unsqueeze(( 0)) ,
}
return tensor_data
def postprocessing(text):
out_text =(
 text.replace(" ` ", "`")
.replace("* * * *", "****")
)
 return out_text
def test(row):
if row['sentiment'] == 'neutral':
predicted_text = row['text']
else:
data = process_test_data(row)
 input_ids = data['input_ids'].to(device)
 token_type_ids = data['token_type_ids'].to(device)
 attention_mask = data['attention_mask'].to(device)
 with torch.no_grad() :
start_logits_0, end_logits_0 = model0(input_ids, attention_mask, token_type_ids)
 start_logits_1, end_logits_1 = model1(input_ids, attention_mask, token_type_ids)
 start_logits_2, end_logits_2 = model2(input_ids, attention_mask, token_type_ids)
 start_logits_3, end_logits_3 = model3(input_ids, attention_mask, token_type_ids)
 start_logits_4, end_logits_4 = model4(input_ids, attention_mask, token_type_ids)
 start_logits =(
 start_logits_0
+ start_logits_1
+ start_logits_2
+ start_logits_3
+ start_logits_4 
)/ 5
end_logits =(
 end_logits_0
+ end_logits_1
+ end_logits_2
+ end_logits_3
+ end_logits_4 
)/ 5
predicted_start_idx = torch.argmax(torch.softmax(start_logits, dim=1), dim=1 ).item()
predicted_end_idx = torch.argmax(torch.softmax(end_logits, dim=1), dim=1 ).item()
input_ids = input_ids.squeeze()
predicted_text = tokenizer.decode(input_ids[predicted_start_idx : predicted_end_idx + 1], skip_special_tokens=True)
 predicted_text = postprocessing(predicted_text)
 return predicted_text
preds = final_model.predict(data_test)
 print(preds.shape)
 print(data_test.shape )<save_to_csv>
test_dataset = pd.read_csv('/kaggle/input/tweet-sentiment-extraction/test.csv')
 test_dataset['predicted_text'] = test_dataset.progress_apply(test, axis=1 )
<import_modules><EOS> 
submission = pd.read_csv('/kaggle/input/tweet-sentiment-extraction/sample_submission.csv')
 submission['selected_text'] = test_dataset['predicted_text']
submission.to_csv('./submission.csv', index=False )