Kaggle competition: Titanic - Machine Learning from Disaster
from ignite.engine import Engine, Events, create_supervised_evaluator
from ignite.metrics import RunningAverage, Accuracy, Precision, Recall, Loss, TopKCategoricalAccuracy
from ignite.contrib.handlers import TensorboardLogger
from ignite.contrib.handlers.tensorboard_logger import OutputHandler, OptimizerParamsHandler<compute_test_metric>
train = pd.read_csv('/kaggle/input/titanic/train.csv')
 test = pd.read_csv('/kaggle/input/titanic/test.csv')
 train_final = pd.read_csv('/kaggle/input/titanic/train.csv')
 test_final = pd.read_csv('/kaggle/input/titanic/test.csv' )
def qw_kappa(pred, y):
return cohen_kappa_score(torch.argmax(pred[...,:-1], dim=1 ).cpu().numpy() ,
y.cpu().numpy() ,
weights='quadratic')
 def cl_accuracy(pred, y):
return accuracy_score(torch.argmax(pred[...,:-1], dim=1 ).cpu().numpy() ,
y.cpu().numpy())
 trainer = Engine(update_fn)
 metrics = {
'Loss': Loss(criterion),
'Accuracy': Loss(cl_accuracy),
'ClKappa': Loss(qw_kappa),
'RgKappa': Loss(lambda pred, y: cont_kappa(pred[...,-1], y)) ,
}
evaluator = create_supervised_evaluator(model, metrics=metrics, device=device, non_blocking=True)
 class Metrics(RunningAverage):
def __init__(self, evaluator, output_transform=None, interactive=None):
super().__init__(alpha=0.9, output_transform=output_transform)
 self._evaluator = evaluator
self.interactive = interactive
self.validation_history = {}
self.loss_history = []
def attach(self, engine, name):
super().attach(engine, name)
 engine.add_event_handler(Events.EPOCH_COMPLETED, self.run_evaluation)
 def compute(self):
loss = super().compute()
self.loss_history.append(loss)
 return loss
def run_evaluation(self, engine=None):
self._evaluator.run(eval_train_loader)
 if self.interactive:
print(evaluator.state.metrics)
 for k,v in self._evaluator.state.metrics.items() :
if k not in self.validation_history.keys() :
self.validation_history[k] = [v]
else:
self.validation_history[k].append(v)
 history = Metrics(evaluator, output_transform=lambda out: out['batchloss'], interactive=is_interactive())
 history.attach(trainer, "batchloss" )<train_model>
train = train.drop(columns= ['Name','Ticket','Cabin'])
 test = test.drop(columns= ['Name','Ticket','Cabin'] )
log_path = "./log"
tb_logger = TensorboardLogger(log_dir=log_path)
 tb_logger.attach(trainer,
log_handler=OutputHandler('training', ['batchloss', ]),
event_name=Events.ITERATION_COMPLETED )<init_hyperparams>
train['Embarked_S'] =(train['Embarked'] == 'S' ).astype(int)
 train['Embarked_C'] =(train['Embarked'] == 'C' ).astype(int)
 train['Embarked_Q'] =(train['Embarked'] == 'Q' ).astype(int)
 train['Gender'] =(train['Sex'] == 'male' ).astype(int )
weight_decay = 1e-4
cycle_mult = 2
sim_epochs = 31
epoch_size = len(train_loader)
 lr_sched_params = {'param_name':'lr', 'cycle_size':epoch_size, 'cycle_mult':cycle_mult,
'start_value_mult':0.8, 'end_value_mult':0.25}
mom_sched_params = {'param_name':'momentum', 'cycle_size':epoch_size, 'cycle_mult':cycle_mult,
'start_value':0.85, 'end_value':0.96}
wd_sched_params = {'param_name':'weight_decay', 'cycle_size':epoch_size*sim_epochs, 'cycle_mult':cycle_mult,
'start_value':weight_decay/10, 'end_value':weight_decay*10}
lr = 0.85
body_sched_params = {**lr_sched_params, 'start_value':lr/10, 'end_value':lr/100}
body_sched = CosineAnnealingScheduler(optimizer.param_groups[0], **body_sched_params)
 feat_sched_params = {**lr_sched_params, 'start_value':lr/5, 'end_value':lr/50}
feat_sched = CosineAnnealingScheduler(optimizer.param_groups[1], **feat_sched_params)
 head_sched_params = {**lr_sched_params, 'start_value':lr, 'end_value':lr/20}
head_sched = CosineAnnealingScheduler(optimizer.param_groups[2], **feat_sched_params)
 mom_sched = CosineAnnealingScheduler(optimizer, **mom_sched_params)
 wd_sched = CosineAnnealingScheduler(optimizer, **wd_sched_params)
 schedulers = [body_sched, feat_sched, head_sched, mom_sched, wd_sched]
names = ["lr(body)", "lr(feat)", "lr(head)", "momentum", "wd"]
lr_values0 = np.array(body_sched.simulate_values(num_events=epoch_size*sim_epochs, **body_sched_params))
lr_values1 = np.array(feat_sched.simulate_values(num_events=epoch_size*sim_epochs, **feat_sched_params))
lr_values2 = np.array(head_sched.simulate_values(num_events=epoch_size*sim_epochs, **head_sched_params))
wd_values = np.array(wd_sched.simulate_values(num_events=epoch_size*sim_epochs, **wd_sched_params))
mom_values = np.array(mom_sched.simulate_values(num_events=epoch_size*sim_epochs, **mom_sched_params))
fig = plt.figure(figsize=(16, 4))
ax = plt.subplot()
plt.title(f"Cosine annealing with start_value_mult={lr_sched_params['start_value_mult']}")
 ax.plot(lr_values0[:, 0], lr_values0[:, 1], label=names[0])
 ax.plot(lr_values1[:, 0], lr_values1[:, 1], label=names[1])
 ax.plot(lr_values2[:, 0], lr_values2[:, 1], label=names[2])
 ax.plot(wd_values[:, 0], wd_values[:, 1], label=names[4])
 ax.set_yscale('log')
 ax.set_xlabel('Batches processed')
 ax.set_ylabel("learning rate")
 ax.legend(frameon=False, loc='upper right')
 ax2 = ax.twinx()
ax2.plot(mom_values[:, 0], mom_values[:, 1], label=names[3])
 ax2.set_ylabel("momentum")
 ax2.legend(frameon=False, loc='lower right')
 _ = ax.plot()<train_on_grid>
test['Embarked_S'] =(test['Embarked'] == 'S' ).astype(int)
 test['Embarked_C'] =(test['Embarked'] == 'C' ).astype(int)
 test['Embarked_Q'] =(test['Embarked'] == 'Q' ).astype(int)
 test['Gender'] =(test['Sex'] == 'male' ).astype(int )
scheduler = ParamGroupScheduler(schedulers=schedulers, names=names)
 trainer.add_event_handler(Events.ITERATION_STARTED, scheduler )<train_model>
train = train.drop(columns = ['Sex'])
 test = test.drop(columns = ['Sex'] )
tb_logger.attach(trainer,
log_handler=OptimizerParamsHandler(optimizer, "lr"),
event_name=Events.EPOCH_STARTED )<set_options>
train = train.drop(columns = ['Embarked'])
 test = test.drop(columns = ['Embarked'] )
def setup_logger(logger):
handler = logging.StreamHandler()
formatter = logging.Formatter("%(asctime)s %(name)-12s %(levelname)-8s %(message)s")
 handler.setFormatter(formatter)
 logger.addHandler(handler)
 logger.setLevel(logging.INFO )<choose_model_class>
train.isnull().sum()
trainer.add_event_handler(Events.ITERATION_COMPLETED, TerminateOnNan())
 def default_score_fn(engine):
score = engine.state.metrics['ClKappa']
return score
best_model_handler = ModelCheckpoint(dirname=log_path,
filename_prefix="best",
n_saved=10,
score_name="ClKappa",
score_function=default_score_fn,
require_empty=False)
 evaluator.add_event_handler(Events.COMPLETED, best_model_handler, {'model': model, })
 @trainer.on(Events.EPOCH_COMPLETED)
 @evaluator.on(Events.COMPLETED)
 def empty_cuda_cache(engine):
torch.cuda.empty_cache()
gc.collect()<set_options>
train.isnull().sum()
@trainer.on(Events.EPOCH_STARTED)
 def tweak_bn_momenta(engine):
epoch = engine.state.epoch
if epoch <= 6:
momentum = 0.06 / epoch
pbar.log_message(f"setting bn momentum to {momentum}")
 for module in model.modules() :
if isinstance(module, nn.modules.batchnorm._BatchNorm):
module.momentum = momentum<train_model>
train.fillna(0,inplace=True)
 test.fillna(0,inplace=True )
num_epochs = 7 if debug else sim_epochs
state = trainer.run(train_loader, max_epochs=num_epochs )<find_best_params>
X = train.drop(columns = ['Survived'])
 y = train['Survived']
evaluator.state.metrics<find_best_params>
scaler = MinMaxScaler()
scaled_X = scaler.fit_transform(X )
!ls {log_path}
checkpoints = next(os.walk(log_path)) [2]
checkpoints = sorted(filter(lambda f: f.endswith(".pth"), checkpoints))
scores = [c.split('=')[-1][:-4] for c in checkpoints]
best_epoch = np.argmax(scores)
 print(best_epoch, scores)
 if not checkpoints:
print('No weight files in {}'.format(log_path))
else:
model_path = f'efficientNet_{scores[best_epoch]}.pth'
!cp {os.path.join(log_path, checkpoints[best_epoch])} {model_path}
print(model_path)
 !rm {log_path}/*.pth<find_best_params>
from sklearn.model_selection import StratifiedKFold
from sklearn.svm import SVC
best_model = model
best_model.load_state_dict(torch.load(model_path))
best_model = best_model.cuda().eval()<find_best_params>
scores = []
best_svc = SVC(kernel='rbf')
 cv = StratifiedKFold(n_splits =10, random_state=42, shuffle=True)
 for train_index, test_index in cv.split(scaled_X, y):
print("Train Index: ",train_index)
 print("Test Index: ",test_index)
 X_train, X_test, y_train, y_test = scaled_X[train_index], scaled_X[test_index], y[train_index], y[test_index]
best_svc.fit(X_train, y_train)
 scores.append(best_svc.score(X_test, y_test))
with torch.no_grad() :
y_pred = best_model(batch[0][:16].cuda())
 print('Regressor activations:')
 print(activation(y_pred[:16,-1] ).reshape([2,8] ).cpu() , '
')
 print('Predictions for first item:')
 for idx in torch.topk(y_pred[0,:-1], k=n_classes)[1].squeeze(0 ).tolist() :
prob = torch.softmax(y_pred, dim=-1)[0, idx].item()
print('{label:<75}({p:.2f}%)'.format(label=classes[idx], p=prob*100))<drop_column>
print("Overall Score: ",np.mean(scores))
del batch<predict_on_test>
scores_alt = cross_val_score(best_svc, scaled_X, y, cv=10)
 print("Overall Score: ",np.mean(scores_alt))
use_regressor = False
def inference_update_with_tta(engine, batch, use_regressor=use_regressor):
global preds, targets
best_model.eval()
with torch.no_grad() :
x, y = batch
x = x.cuda()
if use_regressor:
y_pred1 = best_model(x)[...,-1]
y_pred2 = best_model(x.flip(dims=(-1,)))[...,-1]
curr_pred =(activation(y_pred1)+ activation(y_pred2)) * 0.5
preds += curr_pred.cpu().squeeze().tolist()
else:
y_pred1 = best_model(x)[...,:-1]
y_pred2 = best_model(x.flip(dims=(-1,)))[...,:-1]
curr_pred = F.softmax(y_pred1, dim=-1)+ F.softmax(y_pred2, dim=-1)
 preds += curr_pred.argmax(dim=-1 ).cpu().squeeze().tolist()
targets += y.cpu().squeeze().tolist()
return y_pred1, y
inferencer = Engine(inference_update_with_tta)
 ProgressBar(desc="Inference" ).attach(inferencer )<train_model>
clf = RandomForestClassifier(n_estimators=1000)
 cbc = CatBoostClassifier(eval_metric = 'Accuracy', random_seed = 42, learning_rate=0.01)
 xgb = XGBClassifier(n_estimators=1000, learning_rate=0.01, max_depth=3)
 lgbm = LGBMClassifier(n_estimators=1000, learning_rate=0.01)
 clf_scores = cross_val_score(clf, scaled_X, y, cv=10)
 cbc_scores = cross_val_score(cbc, scaled_X, y, cv=10)
 xgb_scores = cross_val_score(xgb, scaled_X, y, cv=10)
 lgbm_scores = cross_val_score(lgbm, scaled_X, y, cv=10 )
preds, targets = [], []
result_state = inferencer.run(eval_train_loader, max_epochs=1)
 print('valid accuracy:',(np.array(preds)== np.array(targets)).mean() )<predict_on_test>
print("Random Forest: ",np.mean(clf_scores))
print("CatBoost: ",np.mean(cbc_scores))
print("XGBoost: ",np.mean(xgb_scores))
print("LightGBM: ",np.mean(lgbm_scores))
class KappaOptimizer(nn.Module):
def __init__(self):
super().__init__()
self.coef = [0.5, 1.5, 2.5, 3.5]
self.func = self.quad_kappa
def predict(self, preds):
return self._predict(self.coef, preds)
 @classmethod
def _predict(cls, coef, preds):
if type(preds ).__name__ == 'Tensor':
y_hat = preds.clone().view(-1)
 else:
y_hat = torch.FloatTensor(preds ).view(-1)
 for i,pred in enumerate(y_hat):
if pred < coef[0]: y_hat[i] = 0
elif pred < coef[1]: y_hat[i] = 1
elif pred < coef[2]: y_hat[i] = 2
elif pred < coef[3]: y_hat[i] = 3
else: y_hat[i] = 4
return y_hat.int()
def quad_kappa(self, preds, y):
return self._quad_kappa(self.coef, preds, y)
 @classmethod
def _quad_kappa(cls, coef, preds, y):
y_hat = cls._predict(coef, preds)
 return cohen_kappa_score(y, y_hat, weights='quadratic')
 def fit(self, preds, y):

 print('Early score:', self.quad_kappa(preds, y))
neg_kappa = lambda coef: -self._quad_kappa(coef, preds, y)
 opt_res = sp.optimize.minimize(neg_kappa, x0=self.coef, method='nelder-mead',
options={'maxiter':100, 'fatol':1e-20, 'xatol':1e-20})
 print(opt_res)
 self.coef = opt_res.x
print('New score:', self.quad_kappa(preds, y))
def forward(self, preds, y):

 return torch.tensor(self.quad_kappa(preds, y))
if use_regressor:
kappa_opt = KappaOptimizer()
kappa_opt.fit(preds, targets)
 opt_preds = kappa_opt.predict(preds ).tolist()
_ = pd.DataFrame(preds ).hist()
_ = pd.DataFrame(opt_preds ).hist()<split>
train_final.fillna(-999,inplace=True)
 X = train_final.drop(columns = ['Survived'])
 y = train_final['Survived']
X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=0.85, random_state=1234)
 features = np.where(X.dtypes!=float)[0]
cbc = CatBoostClassifier(eval_metric = 'Accuracy', random_seed = 42, use_best_model=True)
 cbc.fit(X_train, y_train, cat_features=features, eval_set=(X_test, y_test), early_stopping_rounds=100 )
preds, targets = [], []
result_state = inferencer.run(test_loader, max_epochs=1 )<predict_on_test>
pred = cbc.predict(X_test)
 accuracy_score(y_test, pred )
if use_regressor:
preds = kappa_opt.predict(preds ).tolist()<load_from_csv>
test_final.fillna(-999,inplace=True)
 predictions = cbc.predict(test_final)
 predictions
submission = pd.DataFrame({'id_code': pd.read_csv('.. /input/aptos2019-blindness-detection/sample_submission.csv' ).id_code.values,
'diagnosis': np.squeeze(preds ).astype(np.int32)})
 submission.hist()
submission.head()<save_to_csv>
result = pd.DataFrame({'PassengerId':test['PassengerId'],'Survived':predictions} )
submission.to_csv('submission.csv', index=False )<save_to_csv>
result.to_csv("submission.csv", index=False )