Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<train_model>
init_notebook_mode()
config = {}
if ae_epochs > 0:
base = get_base(config)
 ae_model = get_ae_model(base, config)
 for i in range(ae_epochs//ae_epochs_each):
print(f"------ {i} ------")
 print("--- train ---")
 X_node_shuff, As_shuff = shuffle(X_node, As)
 ae_model.fit([X_node_shuff, As_shuff], [np.zeros(( len(X_node)))],
epochs = ae_epochs_each,
batch_size = ae_batch_size)
 print("--- public ---")
 X_node_pub_shuff, As_pub_shuff = shuffle(X_node_pub, As_pub)
 ae_model.fit([X_node_pub_shuff, As_pub_shuff], [np.zeros(( len(X_node_pub)))],
epochs = ae_epochs_each,
batch_size = ae_batch_size)
 print("--- private ---")
 X_node_pri_shuff, As_pri_shuff = shuffle(X_node_pri, As_pri)
 ae_model.fit([X_node_pri_shuff, As_pri_shuff], [np.zeros(( len(X_node_pri)))],
epochs = ae_epochs_each,
batch_size = ae_batch_size)
 gc.collect()
print("****** save ae model ******")
 base.save_weights("base_ae_lstm_lstm" )<set_options>
features1 = ["Age","SibSp","Parch", "Pclass", "is_male", "is_female", "f_size", "Single", "SmallF", "MedF", "LargeF", "Title", "Embarked"]
target = ["Survived"]
print("Total number of features is {}".format(len(features1)) )
del ae_model
gc.collect()<save_model>
test_csv = prepare_data('.. /input/test.csv', True)
 test_data = test_csv[features1]
test_csv.head(5 )
np.save('X_node_pub.npy', X_node_pub)
 np.save('X_node_pri.npy', X_node_pri)
 np.save('As_pub.npy', As_pub)
 np.save('As_pri.npy', As_pri)
 del X_node_pub, X_node_pri, As_pub, As_pri, X_node_shuff, As_shuff, X_node_pub_shuff, As_pub_shuff, X_node_pri_shuff, As_pri_shuff<split>
train, valid = train_test_split(data, test_size=0.2)
 print("The Train Set Size is {}
The Validation Set Size is {}".format(len(train), len(valid)))
 print("Test Set Size is {}".format(len(test_data)) )
kfold = GroupKFold(5)
 config = {}
scores = []
preds = np.zeros([len(X_node), X_node.shape[1], 5])
 X_node, As, y, weights, groups_shuffled, SN_filter_mask = shuffle(X_node, As, y, train.signal_to_noise.values, train['id'],(train['SN_filter'] == 1 ).values)
 del train
for i,(tr_idx, va_idx)in enumerate(kfold.split(X_node, As, groups_shuffled)) :
print(f"------ fold {i} start -----")
 X_node_tr = X_node[tr_idx]
X_node_va = X_node[va_idx]
As_tr = As[tr_idx]
As_va = As[va_idx]
y_tr = y[tr_idx]
y_va = y[va_idx]
w_trn = np.log(weights[tr_idx]+1.105)/2
base = get_base(config)
 if ae_epochs > 0:
print("****** load ae model ******")
 base.load_weights("base_ae_lstm_lstm")
 model = get_model(base, config)
 for epochs, batch_size in zip(epochs_list, batch_size_list):
print(f"epochs : {epochs}, batch_size : {batch_size}")
 model.fit([X_node_tr, As_tr], [y_tr],
validation_data=([X_node_va[SN_filter_mask[va_idx]], As_va[SN_filter_mask[va_idx]]], [y_va[SN_filter_mask[va_idx]]]),
epochs = epochs,
sample_weight=w_trn/2,
batch_size = batch_size, validation_freq = 3)
 model.save_weights(f"model{i}_lstm_lstm")
 p = model.predict([X_node_va, As_va])
 for col in range(5):
print(targets[col],(( p[:, :68, col][SN_filter_mask[va_idx]] - y_va[:,:68,col][SN_filter_mask[va_idx]])** 2 ).mean() **.5)
 scores.append(np.mean(mcrmse_loss(y_va, p)))
 print(f"fold {i}: mcrmse {scores[-1]}")
 preds[va_idx] = p
gc.collect()<create_dataframe>
train_x , train_y = data[features1].as_matrix() , data[target].as_matrix()
clf = DecisionTreeClassifier()
clf.fit(train_x, train_y)
 print(( np.array(clf.predict(valid[features1].as_matrix())== valid[target].as_matrix().flatten() , dtype=np.int ).sum() * 100.) / len(valid))
result = pd.DataFrame(data={'PassengerId': test_csv['PassengerId'], 'Survived': clf.predict(test_data.as_matrix())})
 result.to_csv(path_or_buf='decision_tree_submittion.csv', index = False, header = True )
preds_df = []
for p_ix, _id in zip(range(preds.shape[0]), groups_shuffled):
for i in range(68):
preds_df.append([f'{_id}_{i}', preds[p_ix, i, 0], preds[p_ix, i, 1],
preds[p_ix, i, 3], y[p_ix, i, 0], y[p_ix, i, 1],
y[p_ix, i, 3], SN_filter_mask[p_ix]])
 preds_df = pd.DataFrame(preds_df, columns=['id_seqpos', 'reactivity', 'deg_Mg_pH10',
'deg_Mg_50C', 'reactivity_gt', 'deg_Mg_pH10_gt',
'deg_Mg_50C_gt', 'SN_filter'])
 preds_df = preds_df.groupby('id_seqpos' ).mean()
rmses = []
mses = []
for col in ['reactivity', 'deg_Mg_pH10', 'deg_Mg_50C']:
_rmse =(( preds_df[preds_df['SN_filter'] == 1][col] - preds_df[preds_df['SN_filter'] == 1][col+'_gt'])** 2 ).mean() **.5
_mse =(( preds_df[preds_df['SN_filter'] == 1][col] - preds_df[preds_df['SN_filter'] == 1][col+'_gt'])** 2 ).mean()
rmses.append(_rmse)
 mses.append(_mse)
 print(col, _rmse, _mse)
 print(np.mean(rmses), np.mean(mses))<drop_column>
class TitanicLoader(Dataset):
def __init__(self,train,transforms=None):
self.X = train.as_matrix(columns=features1)
 self.Y = train.as_matrix(columns=target ).flatten()
self.count = len(self.X)
 self.transforms = transforms
def __getitem__(self, index):
nextItem = Variable(torch.tensor(self.X[index] ).type(torch.FloatTensor))
if self.transforms is not None:
nextItem = self.transforms(nextItem[0])
 return(nextItem, self.Y[index])
 def __len__(self):
return self.count
del X_node, As<load_pretrained>
class DNN(nn.Module):
def __init__(self, input_size, first_hidden_size, second_hidden_size, num_classes):
super(DNN, self ).__init__()
self.z1 = nn.Linear(input_size, first_hidden_size)
 self.relu = nn.ReLU()
self.z2 = nn.Linear(first_hidden_size, second_hidden_size)
 self.z3 = nn.Linear(second_hidden_size, num_classes)
 self.bn1 = nn.BatchNorm1d(first_hidden_size)
 self.bn2 = nn.BatchNorm1d(second_hidden_size)
 self.dropout = nn.Dropout(p=0.5)
 self.dropout2 = nn.Dropout(p=0.1)
 self.log_softmax = nn.LogSoftmax(dim=1)
 def forward(self, x):
out = self.z1(x)
 out = self.relu(out)
 out = self.dropout(out)
 out = self.bn1(out)
 out = self.z2(out)
 out = self.relu(out)
 out = self.dropout2(out)
 out = self.bn2(out)
 out = self.z3(out)
 out = self.log_softmax(out)
 return out
def name(self):
return "DNN"
X_node_pub = np.load('X_node_pub.npy')
 X_node_pri = np.load('X_node_pri.npy')
 As_pub = np.load('As_pub.npy')
 As_pri = np.load('As_pri.npy' )<load_pretrained>
def train_dnn(net, trainL, validL):
count = 0
accuList = []
lossList = []
optimizer = torch.optim.Adam(net.parameters() ,lr=0.001)
 for epc in range(1,epochs + 1):
print("Epoch
vcount = 0
total_loss = 0
net.train()
for data,target in trainL:
optimizer.zero_grad()
out = net(data)
 loss = F.nll_loss(out, target, size_average=False)
 pred = out.data.max(1, keepdim=True)[1]
count += pred.eq(target.data.view_as(pred)).sum()
loss.backward()
optimizer.step()
net.eval()
for data, target in validL:
out = net(data)
 loss = F.nll_loss(out, target, size_average=False)
 total_loss += loss.item()
pred = out.data.max(1, keepdim=True)[1]
vcount += pred.eq(target.data.view_as(pred)).sum().item()
accuList.append(100.*(vcount / len(validL)))
 lossList.append(total_loss / len(validL))
return accuList, lossList
def test(net, loader):
net.eval()
vcount = 0
count = 0
total_loss = 0.0
for data, target in loader:
out = net(data)
 loss = F.nll_loss(out, target, size_average=False)
 total_loss += loss.item()
pred = out.data.max(1, keepdim=True)[1]
vcount += pred.eq(target.data.view_as(pred)).sum().item()
return 100.*(vcount / len(loader)) , total_loss / len(loader )
p_pub = 0
p_pri = 0
for i in range(5):
config = {}
base = get_base(config)
 if ae_epochs > 0:
print("****** load ae model ******")
 base.load_weights("base_ae_lstm_lstm")
 model = get_model(base, config)
 model.load_weights(f"model{i}_lstm_lstm")
 p_pub += model.predict([X_node_pub, As_pub])/ 5
p_pri += model.predict([X_node_pri, As_pri])/ 5
for i, target in enumerate(targets):
test_pub[target] = [list(p_pub[k, :, i])for k in range(p_pub.shape[0])]
test_pri[target] = [list(p_pri[k, :, i])for k in range(p_pri.shape[0])]<save_to_csv>
epochs = 12
preds_ls = []
for df, preds in [(test_pub, p_pub),(test_pri, p_pri)]:
for i, uid in enumerate(df.id):
single_pred = preds[i]
single_df = pd.DataFrame(single_pred, columns=targets)
 single_df['id_seqpos'] = [f'{uid}_{x}' for x in range(single_df.shape[0])]
preds_ls.append(single_df)
 preds_df = pd.concat(preds_ls)
 preds_df = preds_df.groupby('id_seqpos' ).mean()
preds_df.to_csv("submission_lstm_lstm.csv")
 preds_df.head()<import_modules>
titanic_train_DS = TitanicLoader(train)
 titanic_valid_DS = TitanicLoader(valid)
 train_loader = torch.utils.data.DataLoader(titanic_train_DS,
batch_size=6, shuffle=False)
 valid_loader = torch.utils.data.DataLoader(titanic_valid_DS,
batch_size=1, shuffle=False )
import numpy as np
import pandas as pd
import os 
<import_modules>
myNet = DNN(len(features1), 23, 4, 2)
 accuList, lossList = train_dnn(myNet, train_loader, valid_loader )
import json
import tensorflow as tf
from matplotlib import pyplot as plt<load_from_csv>
def get_preds(test, net):
net.eval()
preds = []
for data, target in test:
out = net(data)
 pred = out.data.max(1, keepdim=True)[1]
preds.append(pred.item())
 return preds
train_data = pd.read_json('/kaggle/input/stanford-covid-vaccine/train.json', lines = True)
 test_data = pd.read_json('/kaggle/input/stanford-covid-vaccine/test.json', lines = True)
 submission_format = pd.read_csv('/kaggle/input/stanford-covid-vaccine/sample_submission.csv', encoding = 'utf-8-sig' )<groupby>
test_data['Survived'] = -1
titanic_test_DS = TitanicLoader(test_data)
 test_loader = torch.utils.data.DataLoader(titanic_test_DS,
batch_size=1, shuffle=False)
 result = pd.DataFrame(data={'PassengerId': test_csv['PassengerId'], 'Survived': get_preds(test_loader, myNet)})
 result.to_csv(path_or_buf='neural_network_submittion.csv', index = False, header = True )
train_data.groupby(['SN_filter'] ).size()<count_values>
neigh = KNeighborsClassifier(n_neighbors=5, weights='distance', p=1)
 neigh.fit(train[features1].as_matrix() , train[target].as_matrix().flatten())
 print_acc(( np.array(neigh.predict(valid[features1].as_matrix())== valid[target].as_matrix().flatten() , dtype=np.int ).sum() * 100.) / len(valid), "K-NN")
 result = pd.DataFrame(data={'PassengerId': test_csv['PassengerId'], 'Survived': neigh.predict(test_data[features1].as_matrix())})
 result.to_csv(path_or_buf='knn_submittion.csv', index = False, header = True )
print('Training data:
',train_data['seq_scored'].value_counts())
 print('Test data:
',test_data['seq_scored'].value_counts())
 len(train_data['reactivity'].iloc[0] )<count_values>
gnb = GaussianNB()
y_pred = gnb.fit(train[features1].as_matrix() , train[target].as_matrix().flatten() ).predict(valid[features1].as_matrix())
 print_acc(float(np.array(y_pred == valid[target].as_matrix().flatten() , dtype=np.int ).sum() * 100)/ len(valid), "Naive Bayes")
 result = pd.DataFrame(data={'PassengerId': test_csv['PassengerId'], 'Survived': gnb.predict(test_data[features1].as_matrix())})
 result.to_csv(path_or_buf='naive_bayes_submittion.csv', index = False, header = True )
flag = False
for i in range(0,len(train_data)) :
if(( [x<0 for x in train_data['reactivity_error'].iloc[i]].count(True)> 0)| 
([x<0 for x in train_data['deg_error_Mg_pH10'].iloc[i]].count(True)> 0)| 
([x<0 for x in train_data['deg_error_pH10'].iloc[i]].count(True)> 0)| 
([x<0 for x in train_data['deg_error_Mg_50C'].iloc[i]].count(True)> 0)| 
([x<0 for x in train_data['deg_error_50C'].iloc[i]].count(True)> 0)) :
flag = True
print(flag )<define_variables>
clf1 = LogisticRegression(random_state=1)
 clf2 = RandomForestClassifier(n_estimators=25,random_state=1)
 clf3 = GaussianNB(var_smoothing=True)
 clf4 = LinearSVC(random_state=5)
 gbm = xgb.XGBClassifier(max_depth=5, n_estimators=300, learning_rate=0.05)
 eclf1 = VotingClassifier(estimators=[('lr', clf1),('rf', clf2),('gnb', clf3),('xgb', gbm)], voting='soft')
 eclf1 = eclf1.fit(train[features1].as_matrix() , train[target].as_matrix().flatten())
 print_acc(float(np.array(eclf1.predict(valid[features1].as_matrix())== valid[target].as_matrix().flatten() , dtype=np.int ).sum() * 100)/ len(valid), "Voting")
 result = pd.DataFrame(data={'PassengerId': test_csv['PassengerId'], 'Survived': eclf1.predict(test_data[features1].as_matrix())})
 result.to_csv(path_or_buf='soft_voting_submittion.csv', index = False, header = True )
<feature_engineering><EOS> 
gbm = xgb.XGBClassifier(max_depth=3, n_estimators=600, learning_rate=0.05)
 y_pred = gbm.fit(train[features1].as_matrix() , train[target].as_matrix().flatten() ).predict(valid[features1].as_matrix())
 print_acc(float(np.array(y_pred == valid[target].as_matrix().flatten() , dtype=np.int ).sum() * 100)/ len(valid), "XGB")
 result = pd.DataFrame(data={'PassengerId': test_csv['PassengerId'], 'Survived': gbm.predict(test_data[features1].as_matrix())})
 result.to_csv(path_or_buf='gbm_submittion.csv', index = False, header = True )