Kaggle competition: Tweet Sentiment Extraction
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim<define_search_model>
import transformers
import torch
import torch.nn as nn
from torch.utils.data import DataLoader
import math
import plotly.graph_objects as go
from plotly.subplots import make_subplots
from tqdm import tqdm_notebook
import re
import os, sys
import random
def model_lstm_atten(embedding_matrix):
inp1 = Input(shape=(maxlen,), name='inp1')
 inp2 = Input(shape=(nb_features,), name='inp2')
 x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp1)
 x = SpatialDropout1D(0.25 )(x)
 x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)
 x = Attention(maxlen )(x)
 x = concatenate([x, inp2])
 x = Dense(128, activation="relu" )(x)
 x = Dense(64, activation="relu" )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=[inp1, inp2], outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model<predict_on_test>
def seed_everything(seed_value):
random.seed(seed_value)
 np.random.seed(seed_value)
 torch.manual_seed(seed_value)
 os.environ['PYTHONHASHSEED'] = str(seed_value)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed_value)
 torch.cuda.manual_seed_all(seed_value)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
seed = 42
seed_everything(seed )
def train_pred(model, epochs=2):
for e in range(epochs):
model.fit({'inp1': train_X, 'inp2': features}, train_y, batch_size=512, epochs=1,
validation_data=({'inp1': val_X, 'inp2': val_features}, val_y))
pred_val_y = model.predict({'inp1': val_X, 'inp2': val_features}, batch_size=1024, verbose=0)
 pred_test_y = model.predict({'inp1': test_X, 'inp2': test_features}, batch_size=1024, verbose=0)
 return pred_val_y, pred_test_y<split>
os.environ['CUDA_LAUNCH_BLOCKING'] = '1'
train_X, val_X, test_X, train_y, val_y, features, val_features, test_features, word_index = load_and_prec()<save_model>
train = pd.read_csv('/kaggle/input/tweet-sentiment-extraction/train.csv')
 test = pd.read_csv('/kaggle/input/tweet-sentiment-extraction/test.csv' )
np.save("train_X",train_X)
 np.save("val_X",val_X)
 np.save("test_X",test_X)
 np.save("train_y",train_y)
 np.save("val_y",val_y)
 np.save("features",features)
 np.save("val_features",val_features)
 np.save("test_features",test_features)
 np.save("word_index.npy",word_index)
 train_X = np.load("train_X.npy")
 val_X = np.load("val_X.npy")
 test_X = np.load("test_X.npy")
 train_y = np.load("train_y.npy")
 val_y = np.load("val_y.npy")
 features = np.load("features.npy")
 val_features = np.load("val_features.npy")
 test_features = np.load("test_features.npy")
 word_index = np.load("word_index.npy" ).item()<load_pretrained>
train.isnull().sum()
embedding_matrix_1 = load_glove(word_index)
 embedding_matrix_2 = load_fasttext(word_index)
 embedding_matrix_3 = load_para(word_index )<compute_test_metric>
train.isnull().sum()
embedding_matrix = np.mean([embedding_matrix_1, embedding_matrix_2, embedding_matrix_3], axis = 0)
 np.shape(embedding_matrix )<define_variables>
print(len(train))
train = train.dropna()
print(len(train))
outputs = []<predict_on_test>
def compute_jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 c = a.intersection(b)
 if(len(a)==0)&(len(b)==0):
return 0.5
else:
return round(float(len(c)) /(len(a)+ len(b)- len(c)) , 2 )
pred_val_y, pred_test_y = train_pred(model_lstm_atten(embedding_matrix), epochs = EPOCHS)
 outputs.append([pred_val_y, pred_test_y, '2 LSTM w/ attention'] )<compute_test_metric>
compute_jaccard(train.text[3], train.selected_text[3] )
score = 0
thresh =.5
for i in np.arange(0.1, 0.991, 0.01):
y_val =(np.array(pred_val_y)> i ).astype(np.int)
 temp_score = f1_score(val_y, y_val)
 if(temp_score > score):
score = temp_score
thresh = i
print("CV: {}, Threshold: {}".format(score, thresh))<save_to_csv>
dist_score_pos = []
dist_score_neg = []
dist_score_neu = []
group_labels = ['positive', 'negative', 'neutral']
for label in group_labels:
for t, s_t in zip(train.text[train.sentiment == label], train.selected_text[train.sentiment == label]):
score = compute_jaccard(t, s_t)
 if label == 'positive':
dist_score_pos.append(score)
 elif label == 'negative':
dist_score_neg.append(score)
 else:
dist_score_neu.append(score)
 hist_data = [dist_score_pos, dist_score_neg, dist_score_neu]
pred_test_y = np.mean([outputs[i][1] for i in range(len(outputs)) ], axis = 0)
 pred_test_y =(pred_test_y > thresh ).astype(int)
 test_df = pd.read_csv(".. /input/test.csv", usecols=["qid"])
 out_df = pd.DataFrame({"qid":test_df["qid"].values})
 out_df['prediction'] = pred_test_y
out_df.to_csv("submission.csv", index=False )<train_model>
dist_score_neu.count(1)/ len(dist_score_neu )
t_finish = time.time()
print(f"Kernel run time = {(t_finish-t_start)/3600} hours" )<load_from_csv>
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 _train = add_features(train_df)
 _test = add_features(test_df)
 features = _train.fillna(0)
 test_features = _test.fillna(0)
 ss = StandardScaler()
ss.fit(np.vstack(( features, test_features)))
 features = ss.transform(features)
 test_features = ss.transform(test_features)
<load_from_csv>
path_spiece_model = '/kaggle/input/albert-model/albert-base-v2-spiece.model'
path_albert_config = '/kaggle/input/albert-model/albert-base-v2-config.json'
path_albert_model = '/kaggle/input/albert-model/albert-base-v2-pytorch_model.bin'
t_data= pd.read_csv('/kaggle/input/titanic/train.csv',index_col='PassengerId')
 t_data.head()<drop_column>
sp = spm.SentencePieceProcessor()
sp.load(path_spiece_model )
t_data.drop(columns=['Name','Ticket','Fare','Cabin'],inplace=True )<count_values>
tokenizer = transformers.AlbertTokenizer.from_pretrained(path_spiece_model, do_lower_case=True)
 tokenizer.tokenize("Test tokenizer" )
for col in range(len(t_data.columns)) :
print(t_data[t_data.columns[col]].value_counts() )<count_missing_values>
class OffsetTokenizer() :
def __init__(self, path_model=path_spiece_model):
self.spt = sentencepiece_pb2.SentencePieceText()
self.sp = spm.SentencePieceProcessor()
self.sp.load(path_model)
 def encode(self, text, return_tokens=False, lower=True):
if lower:
text = text.lower()
offset = []
ids = []
self.spt.ParseFromString(self.sp.encode_as_serialized_proto(text))
for piece in self.spt.pieces:
offset.append(( piece.begin, piece.end))
ids.append(piece.id)
 if return_tokens:
return sp.encode_as_pieces(text), ids, offset
else:
return ids, offset
t_data.isna().sum()<count_values>
o_tokenizer = OffsetTokenizer()
o_tokenizer.encode("Test tokenizer", return_tokens=False )
t_data.Age.value_counts().mode()<import_modules>
albert_config = transformers.AlbertConfig.from_pretrained(path_albert_config)
 albert_config.output_hidden_states=True
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score<split>
device = 'cuda' if torch.cuda.is_available() else 'cpu'
target_col="Survived"
y = t_data[target_col]
X = t_data[['Pclass','Sex','Age','SibSp','Parch','Embarked']]
X = pd.get_dummies(X)
 train_X, val_X, train_y, val_y = train_test_split(X, y)
 val_X 
<drop_column>
def pad_or_truncate(list_num, max_len, SEP_ID, is_input_ids=False):
if len(list_num)< max_len:
list_num = list_num + [0] *(max_len - len(list_num))
else:
if is_input_ids:
list_num = list_num[:max_len-1] + SEP_ID
else:
return list_num[:max_len]
return list_num
cols_with_missing = [col for col in train_X.columns if train_X[col].isnull().any() ]
red_X_train=train_X.drop(columns=cols_with_missing)
 red_X_val=val_X.drop(columns=cols_with_missing )<predict_on_test>
def issubset_sequence(s_text_ids, text_ids):
s_text_ids = [str(i)for i in s_text_ids]
ptn_s_text_ids = r'\b' + r'\b \b'.join(s_text_ids)+ r'\b'
text_ids = [str(i)for i in text_ids]
text_ids = ' '.join(text_ids)
 if re.search(ptn_s_text_ids, text_ids):
return True
else:
return False
def get_accuracy(n_estimators,max_depth,train_X, val_X, train_y, val_y):
model = RandomForestClassifier(n_estimators=n_estimators, max_depth=max_depth ,random_state=1)
 model.fit(train_X,train_y)
 preds = model.predict(val_X)
 lr_accuracy = accuracy_score(val_y,preds)
 return lr_accuracy<compute_test_metric>
def data_extractor(df, data_test=False, max_len=128, random=True):

 count = 0
sentiment_id = {'positive' : 2221,
'neutral' : 8387,
'negative' : 3682}
CLS_ID = tokenizer.encode('[CLS]', add_special_tokens=False)
 SEP_ID = tokenizer.encode('[SEP]', add_special_tokens=False)
 all_data = []
if random:
df = df.sample(frac=1, random_state=42)
 texts = df.text.tolist()
sentiments = df.sentiment.tolist()
if not data_test:
sel_texts = df.selected_text.tolist()
for idx, row in df.iterrows() :
start_target, end_target = 0, 0
text = ' '+ ' '.join(row.text.lower().split())
 s_text = ' '+ ' '.join(row.selected_text.lower().split())
 sentiment = row.sentiment.lower().strip()
text_ids, offsets = o_tokenizer.encode(text)
 s_text_ids, _ = o_tokenizer.encode(s_text)
 for i in range(len(s_text_ids)) :
if issubset_sequence(s_text_ids, text_ids):
break
else:
s_text_ids = s_text_ids[1:]
if len(s_text_ids)== 0:
s_text_ids = tokenizer.encode(s_text, add_special_tokens=False)
 for i in range(len(s_text_ids)) :
if issubset_sequence(s_text_ids, text_ids):
break
else:
s_text_ids = s_text_ids[:-1]
for i in range(len(text_ids)) :
if text_ids[i:i+len(s_text_ids)] == s_text_ids:
start_target, end_target = i+3, i+len(s_text_ids)+3-1
if(start_target - end_target)== 1:
end_target += 1
elif end_target-3 > len(text_ids):
print(text, '__|__', s_text)
 print(text_ids)
 print(start_target, end_target)
 print(s_text_ids)
 raise ValueError('tes, end_target > length text ids')
 break
input_ids = CLS_ID + [sentiment_id[sentiment]] + SEP_ID + text_ids + SEP_ID
token_type_ids = [0] * 3 + [1] *(len(text_ids)+1)
 attention_mask = [1] * len(input_ids)
 input_ids = pad_or_truncate(input_ids, max_len, SEP_ID, is_input_ids=True)
 token_type_ids = pad_or_truncate(token_type_ids, max_len, SEP_ID)
 attention_mask = pad_or_truncate(attention_mask, max_len, SEP_ID)
 if len(offsets)< max_len:
offsets = offsets +([(0,0)] *(max_len - len(offsets)))
 else:
offsets = offsets[:max_len]
if start_target == 0 or end_target == 0:
print(text, '_|_', s_text)
 print(tokenizer.tokenize(text), '_|_', tokenizer.tokenize(s_text))
print(text_ids)
 print(s_text_ids)
 raise ValueError('tes')
 albert_input = { 'input_ids':torch.tensor(input_ids ).to(device),
'token_type_ids':torch.tensor(token_type_ids ).to(device),
'attention_mask':torch.tensor(attention_mask ).to(device),
'start_targets':torch.tensor(start_target ).to(device),
'end_targets':torch.tensor(end_target ).to(device),
'offsets':torch.tensor(offsets ).to(device),
'original_texts':text,
'sentiments':sentiment,
}
all_data.append(albert_input)
 else:
for idx, row in df.iterrows() :
text = ' '+ ' '.join(row.text.lower().split())
 sentiment = row.sentiment.strip()
text_ids, offsets = o_tokenizer.encode(text)
 input_ids = CLS_ID + [sentiment_id[sentiment]] + SEP_ID + text_ids + SEP_ID
token_type_ids = [0] * 3 + [1] *(len(text_ids)+1)
 attention_mask = [1] * len(input_ids)
 input_ids = pad_or_truncate(input_ids, max_len, SEP_ID, is_input_ids=True)
 token_type_ids = pad_or_truncate(token_type_ids, max_len, SEP_ID)
 attention_mask = pad_or_truncate(attention_mask, max_len, SEP_ID)
 if len(offsets)< max_len:
offsets = offsets +([(0,0)] *(max_len - len(offsets)))
 else:
offsets = offsets[:max_len]
albert_input = { 'input_ids':torch.tensor(input_ids ).to(device),
'token_type_ids':torch.tensor(token_type_ids ).to(device),
'attention_mask':torch.tensor(attention_mask ).to(device),
'offsets':torch.tensor(offsets ).to(device),
'original_texts':text,
'sentiments':sentiment,
}
all_data.append(albert_input)
 return all_data
accuracy=get_accuracy(200,10,red_X_train,red_X_val,train_y,val_y)
 print("Validation accurcy for Random Forest Model: {}".format(accuracy))<rename_columns>
def generate_dataloader(albert_inputs, split=True, val_size=0.2, batch_size=32):
if split:
val_size = math.floor(len(albert_inputs)*val_size)
 train_data = albert_inputs[val_size:]
val_data = albert_inputs[:val_size]
train_dataloader = DataLoader(train_data, batch_size=batch_size)
 val_dataloader = DataLoader(val_data, batch_size=batch_size)
 return train_dataloader, val_dataloader
else:
return DataLoader(albert_inputs, batch_size=batch_size )
my_imputer = SimpleImputer()
imputed_X_train = pd.DataFrame(my_imputer.fit_transform(train_X))
imputed_X_valid = pd.DataFrame(my_imputer.transform(val_X))
imputed_X_train.columns =train_X.columns
imputed_X_valid.columns = val_X.columns 
<compute_test_metric>
albert_inputs = data_extractor(train)
 train_dataloader, val_dataloader = generate_dataloader(albert_inputs, split=True )
accuracy=get_accuracy(1000,10,imputed_X_train,imputed_X_valid,train_y,val_y)
 print("Validation accurcy for Random Forest Model: {}".format(accuracy))<find_best_params>
len(( albert_inputs[0]['offsets'].sum(1)> 0 ).nonzero() )
max_accur=.5
max_dep=0
best_tree_size=0
for maxDepth in range(1,11):
for i in range(10,101,10):
accuracy=get_accuracy(i,maxDepth,imputed_X_train,imputed_X_valid,train_y,val_y)
 if accuracy>max_accur:
max_accur=accuracy
max_dep=maxDepth
best_tree_size=i
print("max accuracy = {} max depth={} best tree size={}".format(max_accur,max_dep,best_tree_size))<find_best_params>
class TweetSelectionModel(transformers.AlbertPreTrainedModel):
def __init__(self, conf, hidden_dim=768):
super(TweetSelectionModel, self ).__init__(conf, hidden_dim)
 self.albert = transformers.AlbertModel.from_pretrained(path_albert_model,
config=conf, from_tf=False)
 self.drop_out = nn.Dropout(0.3)
 self.linear = nn.Linear(hidden_dim*2, 2)
 torch.nn.init.normal_(self.linear.weight, std=0.02)
 def forward(self, ids, token_type_ids, mask):
_, _, out = self.albert(
 ids,
token_type_ids=token_type_ids,
attention_mask=mask, 
)
 out = torch.cat(( out[-1], out[-2]), dim=-1)
 out = self.drop_out(out)
 logits = self.linear(out)
 start_logits, end_logits = logits.split(1, dim=-1)
 start_logits = start_logits.squeeze(-1)
 end_logits = end_logits.squeeze(-1)
 return start_logits, end_logits
max_accur=.5
max_dep=0
best_tree_size=0
for maxDepth in range(1,11):
for i in range(10,101,10):
accuracy=get_accuracy(i,maxDepth,red_X_train,red_X_val,train_y,val_y)
 if accuracy>max_accur:
max_accur=accuracy
max_dep=maxDepth
best_tree_size=i
print("max accuracy = {} max depth={} best tree size={}".format(max_accur,max_dep,best_tree_size))<categorify>
def loss_fn(start_logits, end_logits, start_positions, end_positions):

 loss_fct = nn.CrossEntropyLoss()
start_loss = loss_fct(start_logits, start_positions)
 end_loss = loss_fct(end_logits, end_positions)
 total_loss =(start_loss + end_loss)
 return total_loss
pd.get_dummies(df, prefix=['col1', 'col2'] )<compute_test_metric>
def eval(data, start_outputs, end_outputs):

 scores = []
input_ids = data['input_ids']
start_targets = data['start_targets']
end_targets = data['end_targets']
offsets = data['offsets']
texts = data['original_texts']
sentiments = data['sentiments']
c_wrong_end = 0
start_outputs = torch.argmax(start_outputs, dim=1)
 end_outputs = torch.argmax(end_outputs, dim=1)
 for text, sentiment, offset, s_target, e_target, s_output, e_output in zip(texts, sentiments, offsets, start_targets,
end_targets, start_outputs, end_outputs):
s_target, e_target, s_output, e_output = s_target.item() -3, e_target.item() -3, max(s_output.item() -3, 0), max(e_output.item() -3, 0)
 len_offset = len(( offset.sum(1)> 0 ).nonzero())
 offset = offset.tolist()
if s_output > e_output:
if sentiment == 'neutral':
e_output = len_offset
else:
s_output = e_output
c_wrong_end += 1
elif e_output > len_offset:
e_output = len_offset
s_char, e_char, s_char_output, e_char_output = offset[s_target][0], offset[e_target][1], offset[s_output][0], offset[e_output][1]
s_text = text[s_char:e_char]
pred_s_text = text[s_char_output:e_char_output]
score_jaccard = compute_jaccard(s_text, pred_s_text)
 scores.append(score_jaccard)
 return c_wrong_end, round(sum(scores)/ len(scores), 2)
accuracy=get_accuracy(60,4,red_X_train,red_X_val,train_y,val_y)
 accuracy<load_from_csv>
def eval_dataloader(model, dataloader):
model.eval()
total_c_wrong_end, jaccard_mean_score, loss_score = 0, 0, 0
for data in dataloader:
input_ids = data['input_ids'].to(device)
 token_type_ids = data['token_type_ids'].to(device)
 attention_mask = data['attention_mask'].to(device)
 start_targets = data['start_targets'].to(device)
 end_targets = data['end_targets'].to(device)
 with torch.no_grad() :
start_outputs, end_outputs = model(input_ids, token_type_ids, attention_mask)
 loss = loss_fn(start_outputs, end_outputs, start_targets, end_targets)
 c_wrong_end, jaccard_score = eval(data, start_outputs, end_outputs)
 total_c_wrong_end += c_wrong_end
if loss_score == 0:
loss_score = loss.item()
else:
loss_score = round(( loss_score + loss.item())/ 2, 3)
 if jaccard_mean_score == 0:
jaccard_mean_score = jaccard_score
else:
jaccard_mean_score = round(( jaccard_score + jaccard_mean_score)/ 2, 3)
 return total_c_wrong_end, loss_score, jaccard_mean_score
test_data= pd.read_csv('/kaggle/input/titanic/test.csv')
 test_data.info() 
<train_model>
def train_model(train_dataloader, val_dataloader, model, epochs=3):
model = model.to(device)
 optimizer = transformers.AdamW(model.parameters() , weight_decay=0.001, lr=1e-5)
 for i in range(epochs):
model.train()
jaccard_mean_score, loss_score = 0, 0
total_c_wrong_end = 0
for data in train_dataloader:
input_ids = data['input_ids'].to(device)
 token_type_ids = data['token_type_ids'].to(device)
 attention_mask = data['attention_mask'].to(device)
 start_targets = data['start_targets'].to(device)
 end_targets = data['end_targets'].to(device)
 model.zero_grad()
start_outputs, end_outputs = model(input_ids, token_type_ids, attention_mask)
 loss = loss_fn(start_outputs, end_outputs, start_targets, end_targets)
 loss.backward()
optimizer.step()
c_wrong_end, jaccard_score = eval(data, start_outputs, end_outputs)
 total_c_wrong_end += c_wrong_end
if loss_score == 0:
loss_score = loss.item()
else:
loss_score = round(( loss_score + loss.item())/ 2, 3)
 if jaccard_mean_score == 0:
jaccard_mean_score = jaccard_score
else:
jaccard_mean_score = round(( jaccard_score + jaccard_mean_score)/ 2, 3)
 val_total_c_wrong_end, val_loss, val_jaccard_mean_score = eval_dataloader(model, val_dataloader)
 print('epoch:', i, end=' ')
 print('train end < start:', str(total_c_wrong_end)+ '/' + str(len(train_dataloader.dataset)) , end=' ')
 print('val end < start:', str(val_total_c_wrong_end)+ '/' + str(len(val_dataloader.dataset)) , end=' ')
 print('train loss:', loss_score, end=' ')
 print('val loss:', val_loss, end=' ')
 print('train jaccard:', jaccard_score, end=' ')
 print('val jaccard:', val_jaccard_mean_score, end=' ')
 print()
return model
model = RandomForestClassifier(n_estimators=60, max_depth=10 ,random_state=1)
 model.fit(imputed_X_train,train_y)
 preds = model.predict(imputed_X_valid)
 model_accuracy = accuracy_score(val_y,preds)
 print("Accarany = {}:".format(model_accuracy))<categorify>
model = TweetSelectionModel(albert_config )
test=test_data[['Pclass','Sex','Age','SibSp','Parch','Embarked']]
final_X_test = pd.get_dummies(test)
 X_test.info()<create_dataframe>
%%time
model = train_model(train_dataloader, val_dataloader, model, epochs=5 )
final_X_test = pd.DataFrame(my_imputer.transform(final_X_test))<save_to_csv>
model.eval()
model.to(device)
 data = next(iter(val_dataloader))
start, stop = model(data['input_ids'], data['token_type_ids'], data['attention_mask'])
 print(start.argmax(1), torch.argmax(stop, dim=1))
print(data['start_targets'], data['end_targets'] )
predictions = model.predict(final_X_test)
 output = pd.DataFrame({'PassengerId': test_data.PassengerId, 'Survived': predictions})
 output.to_csv('my_submission.csv', index=False)
 print("Your submission was successfully saved!")
<load_from_csv>
test_albert_input = data_extractor(test, data_test=True, random=False )
train_data = pd.read_csv("/kaggle/input/titanic/train.csv")
<load_from_csv>
def predict_submission(df, model, device='cpu'):
test_albert_input = data_extractor(df, data_test=True, random=False)
 dataloader = generate_dataloader(test_albert_input, split=False)
 model.eval()
model.to(device)
 predicts = []
s_outputs, e_outputs, a_s_outputs, a_e_outputs, s_chars, e_chars = [], [], [], [], [], []
for data in dataloader:
input_ids = data['input_ids'].to(device)
 token_type_ids = data['token_type_ids'].to(device)
 attention_mask = data['attention_mask'].to(device)
 offsets = data['offsets'].to(device)
 texts = data['original_texts']
sentiments = data['sentiments']
with torch.no_grad() :
start_outputs, end_outputs = model(input_ids, token_type_ids, attention_mask)
 start_outputs, end_outputs = start_outputs.argmax(1), end_outputs.argmax(1)
 for text, sentiment, s_output, e_output, offset in zip(texts, sentiments, start_outputs, end_outputs, offsets):
s_output, e_output = max(s_output.item() -3, 0), max(e_output.item() -3, 0)
 len_offset = len(( offset.sum(1)> 0 ).nonzero())
 s_outputs.append(s_output)
 e_outputs.append(e_output)
 offset = offset.tolist()
if s_output > e_output:
if sentiment == 'neutral':
e_output = len_offset
else:
s_output = e_output
elif e_output > len_offset:
e_output = len_offset
s_char_output, e_char_output = offset[s_output][0], offset[e_output][1]
predict = text[s_char_output:e_char_output]
predicts.append(predict)
 s_chars.append(s_char_output)
 e_chars.append(e_char_output)
 a_s_outputs.append(s_output)
 a_e_outputs.append(e_output)
 return s_outputs, e_outputs, a_s_outputs, a_e_outputs, s_chars, e_chars, predicts
test_data = pd.read_csv("/kaggle/input/titanic/test.csv")
<data_type_conversions>
_, _, _, _, _, _, predicts = predict_submission(test, model, device=device)
 len(predicts), len(test )
all_data['Embarked'].fillna(all_data['Embarked'].mode() [0], inplace = True)
 all_data['Fare'].fillna(all_data['Fare'].median() , inplace = True )<feature_engineering>
test['selected_text'] = predicts
test.head()
all_data['Title'] = all_data.Name.str.extract('([A-Za-z]+)\.', expand=False)
 all_data['Title'].value_counts()
frequent_titles = all_data['Title'].value_counts() [:5].index.tolist()
frequent_titles
all_data['Title'] = all_data['Title'].apply(lambda x: x if x in frequent_titles else 'Other')
 all_data['Title']<feature_engineering>
test[['textID','selected_text']].to_csv('submission.csv', index=False )
median_ages = {}
for title in frequent_titles:
median_ages[title] = all_data.loc[all_data['Title'] == title]['Age'].median()
median_ages['Other'] = all_data['Age'].median()
all_data.loc[all_data['Age'].isnull() , 'Age'] = all_data[all_data['Age'].isnull() ]['Title'].map(median_ages)
 all_data['Age']<categorify>
pd.set_option('display.max_colwidth', -1 )
Cat_Features = ['Sex', 'Embarked', 'Title']
for feature in Cat_Features:
label = LabelEncoder()
all_data[feature] = label.fit_transform(all_data[feature])
 all_data[Cat_Features]<categorify>
train['ori_s_target'] = ori_s_targets
train['ori_e_target'] = ori_e_targets
Cont_Features = ['Age', 'Fare']
num_bins = 5
for feature in Cont_Features:
bin_feature = feature + 'Bin'
all_data[bin_feature] = pd.qcut(all_data[feature], num_bins)
 label = LabelEncoder()
all_data[bin_feature] = label.fit_transform(all_data[bin_feature])
 all_data.head(10 )<feature_engineering>
train = train[:5000]
all_data['Surname'] = all_data.Name.str.extract(r'([A-Za-z]+),', expand=False)
 all_data['TicketPrefix'] = all_data.Ticket.str.extract(r' (.*\d)', expand=False)
 all_data['Surname_Ticket'] = all_data['Surname'] + all_data['TicketPrefix']
all_data['IsFamily'] = all_data.Surname_Ticket.duplicated(keep=False ).astype(int)
 all_data['Child'] = all_data.Age.map(lambda x: 1 if x <=16 else 0)
 FamilyWithChild = all_data[(all_data.IsFamily==1)&(all_data.Child==1)]['Surname_Ticket'].unique()
all_data['FamilyId'] = 0
for ind, identifier in enumerate(FamilyWithChild):
all_data.loc[all_data.Surname_Ticket==identifier, ['FamilyId']] = ind + 1
all_data['FamilySurvival'] = 1
Survived_by_FamilyId = all_data.groupby('FamilyId' ).Survived.sum()
for i in range(1, len(FamilyWithChild)+1):
if Survived_by_FamilyId[i] >= 1:
all_data.loc[all_data.FamilyId==i, ['FamilySurvival']] = 2
elif Survived_by_FamilyId[i] == 0:
all_data.loc[all_data.FamilyId==i, ['FamilySurvival']] = 0
all_data.head(10 )<data_type_conversions>
s_outputs, e_outputs, a_s_outputs, a_e_outputs, s_chars, e_chars, t_predicts = predict_submission(train, model, device=device)
 train['predict'] = t_predicts
train['s_target'] = s_outputs
train['e_target'] = e_outputs
train['a_s_target'] = s_outputs
train['a_e_target'] = e_outputs
train['s_char'] = s_chars
train['e_char'] = e_chars
<prepare_x_and_y><EOS> 
train[train.sentiment == 'positive'][50:100]