Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<set_options>
%matplotlib inline
sns.set_style('darkgrid')
warnings.filterwarnings('ignore' )<set_options>
train=pd.read_csv('.. /input/titanic/train.csv')
 test=pd.read_csv('.. /input/titanic/test.csv' )
def seed_everything(seed_value):
random.seed(seed_value)
 np.random.seed(seed_value)
 torch.manual_seed(seed_value)
 os.environ['PYTHONHASHSEED'] = str(seed_value)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed_value)
 torch.cuda.manual_seed_all(seed_value)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
seed = 42
seed_everything(seed )<feature_engineering>
all_data = pd.concat([train.drop(['Survived'],axis=1),test])
 combine_data = [train,test]
class TweetDataset(torch.utils.data.Dataset):
def __init__(self, df, max_len=96):
self.df = df
self.max_len = max_len
self.labeled = 'selected_text' in df
self.tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file='.. /input/roberta-base/vocab.json',
merges_file='.. /input/roberta-base/merges.txt',
lowercase=True,
add_prefix_space=True)
 def __getitem__(self, index):
data = {}
row = self.df.iloc[index]
ids, masks, tweet, offsets = self.get_input_data(row)
 data['ids'] = ids
data['masks'] = masks
data['tweet'] = tweet
data['offsets'] = offsets
if self.labeled:
start_idx, end_idx = self.get_target_idx(row, tweet, offsets)
 data['start_idx'] = start_idx
data['end_idx'] = end_idx
return data
def __len__(self):
return len(self.df)
 def get_input_data(self, row):
tweet = " " + " ".join(row.text.lower().split())
 encoding = self.tokenizer.encode(tweet)
 sentiment_id = self.tokenizer.encode(row.sentiment ).ids
ids = [0] + sentiment_id + [2, 2] + encoding.ids + [2]
offsets = [(0, 0)] * 4 + encoding.offsets + [(0, 0)]
pad_len = self.max_len - len(ids)
 if pad_len > 0:
ids += [1] * pad_len
offsets += [(0, 0)] * pad_len
ids = torch.tensor(ids)
 masks = torch.where(ids != 1, torch.tensor(1), torch.tensor(0))
offsets = torch.tensor(offsets)
 return ids, masks, tweet, offsets
def get_target_idx(self, row, tweet, offsets):
selected_text = " " + " ".join(row.selected_text.lower().split())
 len_st = len(selected_text)- 1
idx0 = None
idx1 = None
for ind in(i for i, e in enumerate(tweet)if e == selected_text[1]):
if " " + tweet[ind: ind+len_st] == selected_text:
idx0 = ind
idx1 = ind + len_st - 1
break
char_targets = [0] * len(tweet)
 if idx0 != None and idx1 != None:
for ct in range(idx0, idx1 + 1):
char_targets[ct] = 1
target_idx = []
for j,(offset1, offset2)in enumerate(offsets):
if sum(char_targets[offset1: offset2])> 0:
target_idx.append(j)
 start_idx = target_idx[0]
end_idx = target_idx[-1]
return start_idx, end_idx
def get_train_val_loaders(df, train_idx, val_idx, batch_size=8):
train_df = df.iloc[train_idx]
val_df = df.iloc[val_idx]
train_loader = torch.utils.data.DataLoader(
 TweetDataset(train_df),
batch_size=batch_size,
shuffle=True,
num_workers=2,
drop_last=True)
 val_loader = torch.utils.data.DataLoader(
 TweetDataset(val_df),
batch_size=batch_size,
shuffle=False,
num_workers=2)
 dataloaders_dict = {"train": train_loader, "val": val_loader}
return dataloaders_dict
def get_test_loader(df, batch_size=32):
loader = torch.utils.data.DataLoader(
 TweetDataset(df),
batch_size=batch_size,
shuffle=False,
num_workers=2)
 return loader<load_pretrained>
vacant_data = all_data.isnull().sum().sort_values(ascending=False)
 total_data = all_data.isnull().count().sort_values(ascending=False)
 vacant_percent =(vacant_data/total_data)
 missing_df = pd.concat([vacant_data,vacant_percent],axis=1,keys=['Total','Percent'])
 missing_df.head()
class TweetModel(nn.Module):
def __init__(self):
super(TweetModel, self ).__init__()
config = RobertaConfig.from_pretrained(
 '.. /input/roberta-base/config.json', output_hidden_states=True)
 self.roberta = RobertaModel.from_pretrained(
 '.. /input/roberta-base/pytorch_model.bin', config=config)
 self.dropout = nn.Dropout(0.5)
 self.fc = nn.Linear(config.hidden_size, 2)
 nn.init.normal_(self.fc.weight, std=0.02)
 nn.init.normal_(self.fc.bias, 0)
 def forward(self, input_ids, attention_mask):
_, _, hs = self.roberta(input_ids, attention_mask)
 x = torch.stack([hs[-1], hs[-2], hs[-3]])
 x = torch.mean(x, 0)
 x = self.dropout(x)
 x = self.fc(x)
 start_logits, end_logits = x.split(1, dim=-1)
 start_logits = start_logits.squeeze(-1)
 end_logits = end_logits.squeeze(-1)
 return start_logits, end_logits<compute_test_metric>
for dataset in combine_data:
dataset['family_members']= dataset['SibSp']+dataset['Parch']+1
train.head()
def loss_fn(start_logits, end_logits, start_positions, end_positions):
ce_loss = nn.CrossEntropyLoss()
start_loss = ce_loss(start_logits, start_positions)
 end_loss = ce_loss(end_logits, end_positions)
 total_loss = start_loss + end_loss
return total_loss<statistical_test>
for dataset in combine_data:
dataset['Title']=dataset['Name'].str.extract('([A-Za-z]+)\.')
 common_title = []
for title in train['Title'].unique() :
if train[train['Title'] == title]['Title'].count() > 20:
common_title.append(title)
 print(common_title)
 def id_rare(cols):
if cols in common_title:
return cols
else:
return 'Rare'
for dataset in combine_data:
dataset['Title']=dataset['Title'].apply(id_rare)
 title_mapping = {'Mrs':1,'Miss':2,'Master':3,'Mr':4,'Rare':5}
for dataset in combine_data:
dataset['Title']=dataset['Title'].map(title_mapping)
 train.head()
def get_selected_text(text, start_idx, end_idx, offsets):
selected_text = ""
for ix in range(start_idx, end_idx + 1):
selected_text += text[offsets[ix][0]: offsets[ix][1]]
if(ix + 1)< len(offsets)and offsets[ix][1] < offsets[ix + 1][0]:
selected_text += " "
return selected_text
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))
def compute_jaccard_score(text, start_idx, end_idx, start_logits, end_logits, offsets):
start_pred = np.argmax(start_logits)
 end_pred = np.argmax(end_logits)
 if start_pred > end_pred:
pred = text
else:
pred = get_selected_text(text, start_pred, end_pred, offsets)
 true = get_selected_text(text, start_idx, end_idx, offsets)
 return jaccard(true, pred )<train_model>
map_sex = {'male':0,'female':1}
for dataset in combine_data:
dataset['Sex']=dataset['Sex'].map(map_sex )
def train_model(model, dataloaders_dict, criterion, optimizer, num_epochs, filename):
model.cuda()
for epoch in range(num_epochs):
for phase in ['train', 'val']:
if phase == 'train':
model.train()
else:
model.eval()
epoch_loss = 0.0
epoch_jaccard = 0.0
for data in(dataloaders_dict[phase]):
ids = data['ids'].cuda()
masks = data['masks'].cuda()
tweet = data['tweet']
offsets = data['offsets'].numpy()
start_idx = data['start_idx'].cuda()
end_idx = data['end_idx'].cuda()
optimizer.zero_grad()
with torch.set_grad_enabled(phase == 'train'):
start_logits, end_logits = model(ids, masks)
 loss = criterion(start_logits, end_logits, start_idx, end_idx)
 if phase == 'train':
loss.backward()
optimizer.step()
epoch_loss += loss.item() * len(ids)
 start_idx = start_idx.cpu().detach().numpy()
end_idx = end_idx.cpu().detach().numpy()
start_logits = torch.softmax(start_logits, dim=1 ).cpu().detach().numpy()
end_logits = torch.softmax(end_logits, dim=1 ).cpu().detach().numpy()
for i in range(len(ids)) :
jaccard_score = compute_jaccard_score(
 tweet[i],
start_idx[i],
end_idx[i],
start_logits[i],
end_logits[i],
offsets[i])
 epoch_jaccard += jaccard_score
epoch_loss = epoch_loss / len(dataloaders_dict[phase].dataset)
 epoch_jaccard = epoch_jaccard / len(dataloaders_dict[phase].dataset)
 print('Epoch {}/{} | {:^5} | Loss: {:.4f} | Jaccard: {:.4f}'.format(
 epoch + 1, num_epochs, phase, epoch_loss, epoch_jaccard))
torch.save(model.state_dict() , filename )<choose_model_class>
for dataset in combine_data:
dataset['Age']=dataset.groupby(['Sex','Pclass'])['Age'].apply(lambda x : x.fillna(x.median()))
num_epochs = 3
batch_size = 32
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=seed )<load_from_csv>
train['Embarked']=train["Embarked"].fillna('C' )
%%time
train_df = pd.read_csv('.. /input/tweet-sentiment-extraction/train.csv')
 train_df['text'] = train_df['text'].astype(str)
 train_df['selected_text'] = train_df['selected_text'].astype(str)
 for fold,(train_idx, val_idx)in enumerate(skf.split(train_df, train_df.sentiment), start=1):
print(f'Fold: {fold}')
 model = TweetModel()
optimizer = optim.AdamW(model.parameters() , lr=2e-5, betas=(0.9, 0.999))
criterion = loss_fn
dataloaders_dict = get_train_val_loaders(train_df, train_idx, val_idx, batch_size)
 train_model(
 model,
dataloaders_dict,
criterion,
optimizer,
num_epochs,
f'roberta_fold{fold}.pth' )<load_from_csv>
fill_fare = test[(test['Pclass']==3)&(test['Sex']==0)]['Fare'].median()
test['Fare']=test['Fare'].fillna(fill_fare )
%%time
test_df = pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv')
 test_df['text'] = test_df['text'].astype(str)
 test_loader = get_test_loader(test_df)
 predictions = []
models = []
for fold in range(skf.n_splits):
model = TweetModel()
model.cuda()
model.load_state_dict(torch.load(f'roberta_fold{fold+1}.pth'))
model.eval()
models.append(model)
 for data in test_loader:
ids = data['ids'].cuda()
masks = data['masks'].cuda()
tweet = data['tweet']
offsets = data['offsets'].numpy()
start_logits = []
end_logits = []
for model in models:
with torch.no_grad() :
output = model(ids, masks)
 start_logits.append(torch.softmax(output[0], dim=1 ).cpu().detach().numpy())
 end_logits.append(torch.softmax(output[1], dim=1 ).cpu().detach().numpy())
 start_logits = np.mean(start_logits, axis=0)
 end_logits = np.mean(end_logits, axis=0)
 for i in range(len(ids)) :
start_pred = np.argmax(start_logits[i])
 end_pred = np.argmax(end_logits[i])
 if start_pred > end_pred:
pred = tweet[i]
else:
pred = get_selected_text(tweet[i], start_pred, end_pred, offsets[i])
 predictions.append(pred )<save_to_csv>
for dataset in combine_data:
dataset['Age']=dataset['Age'].astype(int)
 for dataset in combine_data:
dataset.loc[ dataset['Age'] <= 16, 'Age'] = 0
dataset.loc[(dataset['Age'] > 16)&(dataset['Age'] <= 32), 'Age'] = 1
dataset.loc[(dataset['Age'] > 32)&(dataset['Age'] <= 48), 'Age'] = 2
dataset.loc[(dataset['Age'] > 48)&(dataset['Age'] <= 64), 'Age'] = 3
dataset.loc[ dataset['Age'] > 64, 'Age'] = 4
sub_df = pd.read_csv('.. /input/tweet-sentiment-extraction/sample_submission.csv')
 sub_df['selected_text'] = predictions
sub_df['selected_text'] = sub_df['selected_text'].apply(lambda x: x.replace('!!!!', '!')if len(x.split())==1 else x)
 sub_df['selected_text'] = sub_df['selected_text'].apply(lambda x: x.replace('.. ', '.')if len(x.split())==1 else x)
 sub_df['selected_text'] = sub_df['selected_text'].apply(lambda x: x.replace('...', '.')if len(x.split())==1 else x)
 sub_df.to_csv('submission.csv', index=False)
 sub_df.head()<import_modules>
for dataset in combine_data:
dataset['is_alone']=0
dataset.loc[dataset['family_members']==1,'is_alone'] = 1
print('TF version',tf.__version__ )<load_from_csv>
map_embarked = {'S':0,'C':1,'Q':2}
for dataset in combine_data:
dataset['Embarked']=dataset['Embarked'].map(map_embarked)
 train.head(2 )
datapath= "/kaggle/input/tweet-sentiment-extraction"
train = pd.read_csv(datapath+"/train.csv" ).fillna('')
 test = pd.read_csv(datapath+"/test.csv" ).fillna('' )<count_missing_values>
train = train.drop(['PassengerId','Name','Ticket','Fare','Cabin','SibSp','Parch'],axis=1)
 test = test.drop(['PassengerId','Name','Ticket','Fare','Cabin','SibSp','Parch'],axis=1 )
print(train[train.isna().any(axis=1)])
 print(test[test.isna().any(axis=1)] )<categorify>
X=train.drop('Survived',axis=1)
 y=train['Survived']
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.30, random_state=100 )
def pre_process(text):

 text = str(text ).lower()
text = re.sub('\[.*?\]', '', text)
 text = re.sub('https?://\S+|www\.\S+', '', text)
 text = re.sub('<.*?>+', '', text)
 text = re.sub('[%s]' % re.escape(string.punctuation), '', text)
 text = re.sub('
', '', text)
 text = re.sub('\w*\d\w*', '', text)
 return text<feature_engineering>
log_model=LogisticRegression()
log_model.fit(X_train,y_train)
 log_model_pred = log_model.predict(X_test)
 acc_log_model = round(log_model.score(X_train,y_train)*100,2)
 print('Accuracy of Logestic Model:',acc_log_model)
 print('--*'*10)
 print(classification_report(y_test,log_model_pred))
print('--*'*10)
 print(confusion_matrix(y_test,log_model_pred))
eda['text2'] = eda['text'].apply(lambda x: pre_process(x))
eda['selectedtxt2'] = eda['selected_text'].apply(lambda x: pre_process(x))
eda.head(10 )<feature_engineering>
tree_model=DecisionTreeClassifier()
tree_model.fit(X_train,y_train)
 tree_model_pred = tree_model.predict(X_test)
 acc_tree_model = round(tree_model.score(X_train,y_train)*100,2)
 print('Accuracy of tree Model:',acc_tree_model)
 print('--*'*10)
 print(classification_report(y_test,tree_model_pred))
print('--*'*10)
 print(confusion_matrix(y_test,tree_model_pred))
test['text2'] = test['text'].apply(lambda x: pre_process(x))
test.head(10 )<define_variables>
rf_model=RandomForestClassifier()
rf_model.fit(X_train,y_train)
 rf_model_pred = rf_model.predict(X_test)
 acc_rf_model = round(rf_model.score(X_train,y_train)*100,2)
 print('Accuracy of Random Foresrt Model:',acc_rf_model)
 print('--*'*10)
 print(classification_report(y_test,rf_model_pred))
print('--*'*10)
 print(confusion_matrix(y_test,rf_model_pred))
model_path = '/kaggle/input/tf-roberta/'
MAX_LEN = 96
tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file=model_path+'vocab-roberta-base.json',
merges_file=model_path+'merges-roberta-base.txt',
lowercase=True,
add_prefix_space=True 
)
 sentiment_id = {'positive': 1313, 'negative': 2430, 'neutral': 7974}<define_variables>
final_log = LogisticRegression()
final_log.fit(X,y)
 final_log_pred = final_log.predict(test)
 final_tree_model=DecisionTreeClassifier()
final_tree_model.fit(X,y)
 final_tree_model_pred = final_tree_model.predict(test)
 final_rf=RandomForestClassifier()
final_rf.fit(X,y)
 final_rf_pred = final_rf.predict(test)
trainrows = train.shape[0]
input_ids = np.ones(( trainrows, MAX_LEN),dtype='int32')
 attention_mask= np.zeros(( trainrows,MAX_LEN),dtype='int32')
 token_type_ids= np.zeros(( trainrows,MAX_LEN),dtype='int32')
 start_tokens = np.zeros(( trainrows,MAX_LEN),dtype='int32')
 end_tokens = np.zeros(( trainrows,MAX_LEN),dtype='int32')
 for k in range(trainrows):
text1 = " "+" ".join(train.loc[k,'text'].split())
 text2 = " ".join(train.loc[k,'selected_text'].split())
 idx = text1.find(text2)
 chars = np.zeros(( len(text1)))
 chars[idx:idx+len(text2)]=1
if text1[idx-1]==' ': chars[idx-1] = 1
enc = tokenizer.encode(text1)
 offsets = []; idx=0
for t in enc.ids:
w = tokenizer.decode([t])
 offsets.append(( idx,idx+len(w)))
 idx += len(w)
 toks = []
for i,(a,b)in enumerate(offsets):
sm = np.sum(chars[a:b])
 if sm>0: toks.append(i)
 s_tok = sentiment_id[train.loc[k,'sentiment']]
input_ids[k,:len(enc.ids)+3] = [0, s_tok] + enc.ids + [2]
attention_mask[k,:len(enc.ids)+3] = 1
if len(toks)>0:
start_tokens[k,toks[0]+2] = 1
end_tokens[k,toks[-1]+2] = 1<categorify>
result= pd.read_csv('.. /input/titanic/gender_submission.csv')
 result=result.set_index('PassengerId')
 result.head(3 )
testrows = test.shape[0]
input_ids_t = np.ones(( testrows, MAX_LEN), dtype='int32')
 attention_mask_t= np.zeros(( testrows, MAX_LEN), dtype='int32')
 token_type_ids_t= np.zeros(( testrows, MAX_LEN), dtype='int32')
 for k in range(test.shape[0]):
text1 = " "+" ".join(test.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 s_tok = sentiment_id[test.loc[k,'sentiment']]
input_ids_t[k,:len(enc.ids)+3] = [0, s_tok] + enc.ids + [2]
attention_mask_t[k,:len(enc.ids)+3] = 1<train_model>
result['Survived']= final_log_pred
result.to_csv('output_log.csv')
 result['Survived']= final_tree_model_pred
result.to_csv('output_tree.csv')
 result['Survived']= final_rf_pred
result.to_csv('output_rf.csv')