Kaggle competition: APTOS 2019 Blindness Detection
<SOS> metric: QuadraticWeightedKappa Kaggle data source: aptos-2019-blindness-detection<define_search_space>
%reload_ext autoreload
%autoreload 2
%matplotlib inline
bounds_LGB = {
'num_leaves':(5, 20),
'min_data_in_leaf':(5, 20),
'learning_rate':(0.01, 0.3),
'min_sum_hessian_in_leaf':(0.00001, 0.01),
'feature_fraction':(0.05, 0.5),
'lambda_l1':(0, 5.0),
'lambda_l2':(0, 5.0),
'min_gain_to_split':(0, 1.0),
'max_depth':(3,15),
}<import_modules>
from fastai import *
from fastai.vision import *
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
import os
import scipy as sp
from functools import partial
from sklearn import metrics
from collections import Counter
from fastai.callbacks import *
import PIL
import cv2
from bayes_opt import BayesianOptimization<choose_model_class>
def seed_everything(seed=1358):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything()
LGB_BO = BayesianOptimization(LGB_bayesian, bounds_LGB, random_state=13 )<define_variables>
PATH = Path('.. /input/aptos2019-blindness-detection' )
init_points = 5
n_iter = 5<find_best_params>
df = pd.read_csv(PATH/'train.csv')
 df.head()
print('-' * 130)
 with warnings.catch_warnings() :
warnings.filterwarnings('ignore')
 LGB_BO.maximize(init_points=init_points, n_iter=n_iter, acq='ucb', xi=0.0, alpha=1e-6 )<init_hyperparams>
df.diagnosis.value_counts()
LGB_BO.probe(
 params={'feature_fraction': 0.1403,
'lambda_l1': 4.218,
'lambda_l2': 1.734,
'learning_rate': 0.07,
'max_depth': 14,
'min_data_in_leaf': 17,
'min_gain_to_split': 0.1501,
'min_sum_hessian_in_leaf': 0.000446,
'num_leaves': 6},
lazy=True,
)<find_best_params>
tfms = get_transforms(do_flip=True, flip_vert=True, max_rotate=0.10, max_zoom=1.3, max_warp=0.0, max_lighting=0.2 )
LGB_BO.maximize(init_points=0, n_iter=0 )<train_model>
data =(
 src.transform(tfms,size=128)
.databunch() 
.normalize(imagenet_stats)
)
 data
for i, res in enumerate(LGB_BO.res):
print("Iteration {}:
\t{}".format(i, res))<init_hyperparams>
def quadratic_kappa(y_hat, y):
return torch.tensor(cohen_kappa_score(torch.round(y_hat), y, weights='quadratic'),device='cuda:0')
 learn = cnn_learner(data, base_arch=models.resnet50 ,metrics=[quadratic_kappa],model_dir='/kaggle',pretrained=True )
param_lgb = {
'num_leaves': int(LGB_BO.max['params']['num_leaves']),
'max_bin': 63,
'min_data_in_leaf': int(LGB_BO.max['params']['min_data_in_leaf']),
'learning_rate': LGB_BO.max['params']['learning_rate'],
'min_sum_hessian_in_leaf': LGB_BO.max['params']['min_sum_hessian_in_leaf'],
'bagging_fraction': 1.0,
'bagging_freq': 5,
'feature_fraction': LGB_BO.max['params']['feature_fraction'],
'lambda_l1': LGB_BO.max['params']['lambda_l1'],
'lambda_l2': LGB_BO.max['params']['lambda_l2'],
'min_gain_to_split': LGB_BO.max['params']['min_gain_to_split'],
'max_depth': int(LGB_BO.max['params']['max_depth']),
'save_binary': True,
'seed': 1337,
'feature_fraction_seed': 1337,
'bagging_seed': 1337,
'drop_seed': 1337,
'data_random_seed': 1337,
'objective': 'binary',
'boosting_type': 'gbdt',
'verbose': 1,
'metric': 'auc',
'is_unbalance': True,
'boost_from_average': False,
}<define_variables>
learn.lr_find()
learn.recorder.plot()
nfold = 5<set_options>
lr = 1e-2
learn.fit_one_cycle(3, lr )
gc.collect()<choose_model_class>
learn.data = data =(
 src.transform(tfms,size=224)
.databunch() 
.normalize(imagenet_stats)
)
 learn.lr_find()
learn.recorder.plot()
skf = StratifiedKFold(n_splits=nfold, shuffle=True, random_state=2019 )<split>
lr = 1e-2
learn.fit_one_cycle(20, lr )
oof = np.zeros(len(train_df))
predictions = np.zeros(( len(test_df),nfold))
i = 1
for train_index, valid_index in skf.split(train_df, train_df.target.values):
print("
fold {}".format(i))
xg_train = lgb.Dataset(train_df.iloc[train_index][predictors].values,
label=train_df.iloc[train_index][target].values,
feature_name=predictors,
free_raw_data = False 
)
 xg_valid = lgb.Dataset(train_df.iloc[valid_index][predictors].values,
label=train_df.iloc[valid_index][target].values,
feature_name=predictors,
free_raw_data = False 
)
 clf = lgb.train(param_lgb, xg_train, 5000, valid_sets = [xg_valid], verbose_eval=250, early_stopping_rounds = 50)
 oof[valid_index] = clf.predict(train_df.iloc[valid_index][predictors].values, num_iteration=clf.best_iteration)
 predictions[:,i-1] += clf.predict(test_df[predictors], num_iteration=clf.best_iteration)
 i = i + 1
print("
CV AUC: {:<0.2f}".format(metrics.roc_auc_score(train_df.target.values, oof)) )<feature_engineering>
learn.unfreeze()
learn.lr_find()
learn.recorder.plot()
print("Rank averaging on", nfold, "fold predictions")
 rank_predictions = np.zeros(( predictions.shape[0],1))
for i in range(nfold):
rank_predictions[:, 0] = np.add(rank_predictions[:, 0], rankdata(predictions[:, i].reshape(-1,1)) /rank_predictions.shape[0])
 rank_predictions /= nfold<create_dataframe>
learn.fit_one_cycle(8, slice(1e-6,1e-3))
sub_df = pd.DataFrame({"ID_code": test_df.ID_code.values})
 sub_df["target"] = rank_predictions
sub_df[:10]<save_to_csv>
valid_preds = learn.get_preds(ds_type=DatasetType.Valid )
sub_df.to_csv("Customer_Transaction_rank_predictions.csv", index=False )<set_options>
class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
ll = metrics.cohen_kappa_score(y, X_p, weights='quadratic')
 return -ll
def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X=X, y=y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 print(-loss_partial(self.coef_['x']))
def predict(self, X, coef):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
return X_p
def coefficients(self):
return self.coef_['x']
warnings.filterwarnings('ignore' )<load_from_csv>
optR = OptimizedRounder()
optR.fit(valid_preds[0],valid_preds[1] )
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv" )<prepare_x_and_y>
sample_df = pd.read_csv(PATH/'sample_submission.csv')
 sample_df.head()
x = train_df.drop(['ID_code', 'target'], axis=1)
 y = train_df['target']
x_test = test_df.drop(['ID_code'], axis =1 )<create_dataframe>
learn.data.add_test(ImageList.from_df(sample_df,PATH,folder='test_images',suffix='.png'))
pd.DataFrame([mean_test, mean_train] )<create_dataframe>
preds,y = learn.get_preds(DatasetType.Test)
pd.DataFrame([std_test, std_train] )<sort_values>
test_predictions = optR.predict(preds, coefficients )
%%time
correlations = x[features].corr().abs().unstack().sort_values(kind="quicksort" ).reset_index()
correlations = correlations[correlations['level_0'] != correlations['level_1']]<set_options>
sample_df.diagnosis = test_predictions.astype(int)
 sample_df.head()
<feature_engineering><EOS> 
sample_df.to_csv('submission.csv',index=False )