Kaggle competition: Titanic - Machine Learning from Disaster
leaks = pd.read_csv('/kaggle/input/leakaggregator/leaked_test_target.csv')
 submission = pd.read_csv('/kaggle/input/bland-by-leak/submission_blend.csv' )<count_missing_values>
train_data_initial = pd.read_csv('.. /input/titanic/train.csv', index_col='PassengerId')
 test_data = pd.read_csv('.. /input/titanic/test.csv', index_col='PassengerId')
 train_data_initial.head()
( ~leaks['meter_reading'].isna() ).sum()<sort_values>
train_data_initial.drop(columns = ['Name','Ticket','Cabin'], inplace=True )
leaks = leaks.sort_values('row_id')
 submission = submission.sort_values('row_id')
 leaks = leaks.set_index(leaks['row_id'])
 submission = submission.set_index(submission['row_id'] )<feature_engineering>
train_data = train_data_initial.copy()
submission.loc[~leaks['meter_reading'].isna() , 'meter_reading'] = leaks.loc[~leaks['meter_reading'].isna() , 'meter_reading']<save_to_csv>
train_data['Age'].fillna(value = train_data['Age'].mean() , inplace = True)
 train_data['Embarked'].fillna(value = train_data['Embarked'].value_counts().idxmax() , inplace = True )
submission.to_csv('bland_with_leaked.csv',index=False )<import_modules>
s =(train_data.dtypes == 'object')
 object_cols = list(s[s].index)
 print("Categorical variables:")
 print(object_cols )
import pandas as pd
import numpy as np<import_modules>
label_train_data = train_data.copy()
label_encoder = LabelEncoder()
for col in object_cols:
label_train_data[col] = label_encoder.fit_transform(train_data[col] )
import pandas as pd
import numpy as np<load_from_csv>
target_col = 'Survived'
y = label_train_data[target_col]
X = label_train_data.drop(columns=[target_col])
 X.head()
fold_1 = pd.read_csv('/kaggle/input/lgbt-fold-1/submission.csv')
 fold_2 = pd.read_csv('/kaggle/input/lgbt-fold-2/submission.csv')
 fold_3 = pd.read_csv('/kaggle/input/lgbt-fold-3/submission.csv')
 fold_4 = pd.read_csv('/kaggle/input/lgbt-fold-4/submission.csv')
 fold_5 = pd.read_csv('/kaggle/input/lgbt-fold-5/submission.csv' )<feature_engineering>
X_train, X_valid, y_train, y_valid = train_test_split(X, y, train_size=0.8, test_size=0.2, random_state=0 )
fold_1['meter_reading'] =(fold_1['meter_reading'] + fold_2['meter_reading'] + fold_3['meter_reading'] + fold_4['meter_reading'] + fold_5['meter_reading'])/5<feature_engineering>
preds_dict = []
for n_estimators in range(100,1001,100):
for max_depth in range(6, 70,10):
for max_leaf_nodes in range(5, 500, 50):
parameters = {'n_estimators': n_estimators,
'max_depth': max_depth,
'max_leaf_nodes': max_leaf_nodes
}
model = RandomForestClassifier(**parameters)
 model.fit(X_train, y_train)
 preds = model.predict(X_valid)
 prediction = {}
prediction['n_estimators'] = n_estimators
prediction['max_depth'] = max_depth
prediction['max_leaf_nodes'] = max_leaf_nodes
prediction['accuracy_score'] = accuracy_score(y_true=y_valid, y_pred=preds)
 preds_dict.append(prediction)
 print(preds_dict )
fold_1['meter_reading'] = np.expm1(fold_1['meter_reading'])
 fold_1.loc[fold_1['meter_reading'] < 0, 'meter_reading'] = 0<save_to_csv>
count = 0
indexIs = 0
maxValue = preds_dict[0]['accuracy_score']
for i in preds_dict:
if maxValue < i['accuracy_score']:
print(count , ' :', i['accuracy_score'])
 maxValue = i['accuracy_score']
indexIs = count
count = count + 1
print(count ,': Max Val is :',maxValue)
 print(preds_dict[indexIs] )
fold_1.to_csv('submission_fold_bland.csv',index=False )<install_modules>
test_data = pd.read_csv('.. /input/titanic/test.csv', index_col='PassengerId' )
!pip install optuna<load_from_csv>
test_data.drop(columns=['Name', 'Ticket','Cabin'], inplace=True)
 test_data
class PreProcess:
def __init__(self):
self.df_otto_train = pd.read_csv('/kaggle/input/otto-group-product-classification-challenge/train.csv')
 self.df_otto_test = pd.read_csv('/kaggle/input/otto-group-product-classification-challenge/test.csv')
 def __call__(self):
conv = lambda x: int(x.replace('Class_', '')) - 1
self.df_otto_train['target2'] = self.df_otto_train.target.map(conv)
 x = self.df_otto_train.drop(['id', 'target', 'target2'], axis=1 ).values
y = self.df_otto_train.target2.values
x_test_id = self.df_otto_test.id.values
x_test = self.df_otto_test.drop(['id'], axis=1 ).values
x_train, x_valid, t_train, t_valid = train_test_split(x, y, test_size=0.2, shuffle=True)
 _, _, _, counts = np.unique(t_train, return_index=True, return_inverse=True, return_counts=True)
 loss_weight =(1 / torch.Tensor(counts)) *(x_train.shape[0]/len(np.unique(t_train)))
 loss_weight = loss_weight.to(device)
 return x_train, x_valid, t_train, t_valid, x_test, x_test_id, loss_weight<choose_model_class>
test_data['Age'].fillna(value = train_data['Age'].mean() , inplace = True)
 test_data['Fare'].fillna(value = train_data['Fare'].mean() , inplace = True )
class MLP(nn.Module):
def __init__(self, in_dim, out_dim):
super(MLP, self ).__init__()
n_layers = 5
layers = []
features = [522, 1006, 1304, 706, 851]
p = [0.1287716033668473, 0.08497812991732616, 0.19230277862176845, 0.09567140996248122, 0.15776969001259095]
in_features = in_dim
for i in range(n_layers):
out_features = features[i]
layers.append(nn.Linear(in_features, out_features))
layers.append(nn.BatchNorm1d(out_features))
layers.append(nn.ReLU())
 layers.append(nn.Dropout(p[i]))
in_features = out_features
layers.append(nn.Linear(in_features, out_dim))
layers.append(nn.LogSoftmax(dim=-1))
self.network = nn.Sequential(*layers)
 def forward(self, x):
return self.network(x)
 class MLP_tune(nn.Module):
def __init__(self, trial, in_dim, out_dim):
super(MLP_tune, self ).__init__()
n_layers = trial.suggest_int("n_layers", 5, 10)
 layers = []
in_features = in_dim
for i in range(n_layers):
out_features = trial.suggest_int("n_units_l{}".format(i), in_dim, in_dim*20)
 layers.append(nn.Linear(in_features, out_features))
layers.append(nn.BatchNorm1d(out_features))
layers.append(nn.ReLU())
 p = trial.suggest_float("dropout_l{}".format(i), 0.0, 0.5)
 layers.append(nn.Dropout(p))
in_features = out_features
layers.append(nn.Linear(in_features, out_dim))
layers.append(nn.LogSoftmax(dim=-1))
self.network = nn.Sequential(*layers)
 def forward(self, x):
return self.network(x )<train_model>
s =(test_data.dtypes == 'object')
 object_cols_test = list(s[s].index)
 print("Categorical variables:")
 print(object_cols_test )
class Train:
def __init__(self):
self.x_train, self.x_valid, self.t_train, self.t_valid, self.x_test, self.x_test_id, self.loss_weight = PreProcess()()
def train(self, n_epochs, lr, weight_decay):
in_dim = self.x_train.shape[1]
out_dim = len(np.unique(self.t_train))
mlp = MLP(in_dim, out_dim ).to(device)
 optimizer = optim.RMSprop(mlp.parameters() , lr=lr, weight_decay=weight_decay)
 scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, 'min')
 criterion = nn.NLLLoss(weight=self.loss_weight)
 ds_train = TensorDataset(torch.tensor(self.x_train ).float() , torch.tensor(self.t_train))
ds_valid = TensorDataset(torch.tensor(self.x_valid ).float() , torch.tensor(self.t_valid))
self.ds_test = TensorDataset(torch.tensor(self.x_test ).float())
 loader_train = DataLoader(ds_train, batch_size=256, shuffle=True)
 loader_valid = DataLoader(ds_valid, batch_size=256, shuffle=True)
 start_time = time.time()
for epoch in range(n_epochs):
losses_train = []
losses_valid = []
preds_train = []
preds_valid = []
trues_train = []
trues_valid = []
mlp.train()
for x, t in loader_train:
true = t.tolist()
trues_train.extend(true)
 mlp.zero_grad()
x = x.to(device)
 t = t.to(device)
 y = mlp.forward(x)
 loss = criterion(y, t.long())
 loss.backward()
optimizer.step()
pred = y.argmax(1 ).tolist()
preds_train.extend(pred)
 losses_train.append(loss.tolist())
 mlp.eval()
for x, t in loader_valid:
true = t.tolist()
trues_valid.extend(true)
 x = x.to(device)
 t = t.to(device)
 y = mlp.forward(x)
 loss = criterion(y, t.long())
 pred = y.argmax(1 ).tolist()
preds_valid.extend(pred)
 losses_valid.append(loss.tolist())
 print('EPOCH: {}, Train [Loss: {:.3f}, F1: {:.3f}, R: {:.3f}, P: {:.3f}], Valid [Loss: {:.3f}, F1: {:.3f}, R: {:.3f}, P: {:.3f}]'.format(
 epoch,
np.mean(losses_train),
f1_score(trues_train, preds_train, average='weighted'),
recall_score(trues_train, preds_train, average='weighted'),
precision_score(trues_train, preds_train, average='weighted'),
np.mean(losses_valid),
f1_score(trues_valid, preds_valid, average='weighted'),
recall_score(trues_valid, preds_valid, average='weighted'),
precision_score(trues_valid, preds_valid, average='weighted')
))
scheduler.step(np.mean(losses_valid))
elapsed_time = time.time() - start_time
print("elapsed_time:{:.1f}".format(elapsed_time / 60)+ "[min]")
 return mlp
def test(self, mlp):
loader_test = DataLoader(self.ds_test, batch_size=256, shuffle=False)
 preds_test = []
mlp.eval()
for x in loader_test:
x = x[0].to(device)
 y = mlp.forward(x)
 y = y.to('cpu' ).detach().numpy()
y = np.exp(y)
 preds_test.extend(y)
 return preds_test
def submit(self, preds_test):
df_sample_submission = pd.read_csv('/kaggle/input/otto-group-product-classification-challenge/sampleSubmission.csv')
 df_submission = pd.DataFrame(columns = df_sample_submission.columns)
 df_submission.id = self.x_test_id
df_submission[df_submission.columns[1:]] = preds_test
df_submission.to_csv('submission.csv', index=False)
 return
def get_optimizer(self, trial, model):
optimizer_names = ['Adam', 'MomentumSGD', 'rmsprop']
optimizer_name = trial.suggest_categorical('optimizer', optimizer_names)
 weight_decay = trial.suggest_loguniform('weight_decay', 1e-10, 1e-3)
 if optimizer_name == optimizer_names[0]:
adam_lr = trial.suggest_loguniform('adam_lr', 1e-5, 1e-1)
 optimizer = optim.Adam(model.parameters() , lr=adam_lr, weight_decay=weight_decay)
 elif optimizer_name == optimizer_names[1]:
momentum_sgd_lr = trial.suggest_loguniform('momentum_sgd_lr', 1e-5, 1e-1)
 optimizer = optim.SGD(model.parameters() , lr=momentum_sgd_lr, momentum=0.9, weight_decay=weight_decay)
 else:
rmsprop_lr = trial.suggest_loguniform('rmsprop_lr', 1e-5, 1e-1)
 optimizer = optim.RMSprop(model.parameters() , lr=rmsprop_lr, weight_decay=weight_decay)
 return optimizer
def objective(self, trial):
self.x_train, self.x_valid, self.t_train, self.t_valid, self.x_test, self.x_test_id, self.loss_weight = PreProcess()()
in_dim = self.x_train.shape[1]
out_dim = len(np.unique(self.t_train))
n_epochs = 100
mlp = MLP_tune(trial, in_dim, out_dim ).to(device)
 optimizer = self.get_optimizer(trial, mlp)
 scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, 'min')
 criterion = nn.NLLLoss(weight=self.loss_weight)
 ds_train = TensorDataset(torch.tensor(self.x_train ).float() , torch.tensor(self.t_train))
ds_valid = TensorDataset(torch.tensor(self.x_valid ).float() , torch.tensor(self.t_valid))
loader_train = DataLoader(ds_train, batch_size=256, shuffle=True)
 loader_valid = DataLoader(ds_valid, batch_size=256, shuffle=True)
 start_time = time.time()
for epoch in range(n_epochs):
losses_train = []
losses_valid = []
preds_train = []
preds_valid = []
trues_train = []
trues_valid = []
mlp.train()
for x, t in loader_train:
true = t.tolist()
trues_train.extend(true)
 mlp.zero_grad()
x = x.to(device)
 t = t.to(device)
 y = mlp.forward(x)
 loss = criterion(y, t.long())
 loss.backward()
optimizer.step()
pred = y.argmax(1 ).tolist()
preds_train.extend(pred)
 losses_train.append(loss.tolist())
 mlp.eval()
for x, t in loader_valid:
true = t.tolist()
trues_valid.extend(true)
 x = x.to(device)
 t = t.to(device)
 y = mlp.forward(x)
 loss = criterion(y, t.long())
 pred = y.argmax(1 ).tolist()
preds_valid.extend(pred)
 losses_valid.append(loss.tolist())
 print('EPOCH: {}, Train [Loss: {:.3f}, F1: {:.3f}, R: {:.3f}, P: {:.3f}], Valid [Loss: {:.3f}, F1: {:.3f}, R: {:.3f}, P: {:.3f}]'.format(
 epoch,
np.mean(losses_train),
f1_score(trues_train, preds_train, average='weighted'),
recall_score(trues_train, preds_train, average='weighted'),
precision_score(trues_train, preds_train, average='weighted'),
np.mean(losses_valid),
f1_score(trues_valid, preds_valid, average='weighted'),
recall_score(trues_valid, preds_valid, average='weighted'),
precision_score(trues_valid, preds_valid, average='weighted')
))
scheduler.step(np.mean(losses_valid))
metric = np.mean(losses_valid)
 trial.report(metric, epoch)
 if trial.should_prune() :
raise optuna.exceptions.TrialPruned()
elapsed_time = time.time() - start_time
print("elapsed_time:{:.1f}".format(elapsed_time / 60)+ "[min]")
 return metric<train_model>
label_test_data = test_data.copy()
label_encoder = LabelEncoder()
for col in object_cols_test:
label_train_data[col] = label_encoder.fit(train_data_initial[col].astype(str))
label_test_data[col] = label_encoder.transform(test_data[col].astype(str))
def run() :
n_cross_valid = 30
preds_test_all = []
for n in range(n_cross_valid):
n_epochs = 100
lr = 7.560243948481283e-05
weight_decay = 0.0005896154168582327
train = Train()
mlp = train.train(n_epochs, lr, weight_decay)
 preds_test = train.test(mlp)
 preds_test_all.append(preds_test)
 preds_test = np.mean(preds_test_all, axis=0)
 train.submit(list(preds_test))
run()<train_model>
final_model = RandomForestClassifier(n_estimators=100, max_leaf_nodes=105, max_depth = 6, random_state= 0)
 final_model.fit(X, y)
 preds = final_model.predict(label_test_data )
def tune() :
train = Train()
study = optuna.create_study(direction="minimize")
 study.optimize(train.objective, n_trials=1000)
 pruned_trials = [t for t in study.trials if t.state == optuna.trial.TrialState.PRUNED]
complete_trials = [t for t in study.trials if t.state == optuna.trial.TrialState.COMPLETE]
print("Study statistics: ")
 print(" Number of finished trials: ", len(study.trials))
print(" Number of pruned trials: ", len(pruned_trials))
print(" Number of complete trials: ", len(complete_trials))
print("Best trial:")
 trial = study.best_trial
print(" Value: ", trial.value)
 print(" Params: ")
 for key, value in trial.params.items() :
print(" {}: {}".format(key, value)) 
<load_from_csv>
sub_df = pd.DataFrame(data={
'PassengerId': label_test_data.index,
'Survived': preds
})
 sub_df.to_csv('submission.csv', index=False)
 print('done' )