Kaggle competition: APTOS 2019 Blindness Detection
<SOS> metric: QuadraticWeightedKappa Kaggle data source: aptos-2019-blindness-detection<drop_column>
%reload_ext autoreload
%autoreload 2
%matplotlib inline
train_ID = train['Id']
test_ID = test['Id']
train.drop(['Id'], axis=1, inplace=True)
 test.drop(['Id'], axis=1, inplace=True)
 train.shape, test.shape<feature_engineering>
from fastai import *
from fastai.vision import *
import pandas as pd
import matplotlib.pyplot as plt
train["SalePrice"] = np.log1p(train["SalePrice"] )<feature_engineering>
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
SEED = 999
seed_everything(SEED )
def outlier_detect(feature, data):
outlier_index = []
for each in feature:
Q1 = np.percentile(data[each], 25)
 Q3 = np.percentile(data[each], 75)
 IQR = Q3 - Q1
min_quartile = Q1 - 1.5*IQR
max_quartile = Q3 + 1.5*IQR
outlier_list = data[(data[each] < min_quartile)|(data[each] > max_quartile)].index
outlier_index.extend(outlier_list)
 outlier_index = Counter(outlier_index)
 outlier_data = list(i for i, n in outlier_index.items() if n > 3)
 return outlier_data<filter>
base_image_dir = os.path.join('.. ', 'input/aptos2019-blindness-detection/')
 train_dir = os.path.join(base_image_dir,'train_images/')
 df = pd.read_csv(os.path.join(base_image_dir, 'train.csv'))
df['path'] = df['id_code'].map(lambda x: os.path.join(train_dir,'{}.png'.format(x)))
 df = df.drop(columns=['id_code'])
 df = df.sample(frac=1 ).reset_index(drop=True)
 df.head(10 )
outlier_data = outlier_detect(['OverallQual','GrLivArea','SalePrice'], train)
 train.loc[outlier_data]<drop_column>
folds = pd.read_csv('.. /input/atposfolds/folds.csv' )
train.drop(train[(train['OverallQual']<5)&(train['SalePrice']>200000)].index, inplace=True)
 train.drop(train[(train['GrLivArea']>4500)&(train['SalePrice']<300000)].index, inplace=True)
 train.reset_index(drop=True, inplace=True )<concatenate>
fold_num = 1
val_idxs = folds[folds['folds'] == fold_num].index.values
train_labels = train['SalePrice'].reset_index(drop=True)
 train_features = train.drop(['SalePrice'], axis=1)
 test_features = test
all_features = pd.concat([train_features, test_features] ).reset_index(drop=True)
 all_features.shape<sort_values>
bs = 64
sz=256
total = all_features.isnull().sum().sort_values(ascending=False)
 percent =(all_features.isnull().sum() /all_features.isnull().count() ).sort_values(ascending=False)
 missing_data = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
 missing_data.head(25 )<data_type_conversions>
tfms = get_transforms(do_flip=True, flip_vert=True, max_rotate=360, max_warp=0,
max_zoom=1.1, max_lighting=0.1, p_lighting=0.5)
 src =(ImageList.from_df(df=df,path='./',cols='path')
.split_by_idx(val_idxs)
.label_from_df(cols='diagnosis', label_cls=FloatList)
)
 data=(src.transform(tfms,size=sz,resize_method=ResizeMethod.SQUISH,padding_mode='zeros')
.databunch(bs=bs,num_workers=4)
.normalize(imagenet_stats)
 )
all_features['MSSubClass'] = all_features['MSSubClass'].apply(str)
 all_features['YrSold'] = all_features['YrSold'].astype(str)
 all_features['MoSold'] = all_features['MoSold'].astype(str )<categorify>
def quadratic_kappa(y_hat, y):
return torch.tensor(cohen_kappa_score(torch.round(y_hat), y, weights='quadratic'),device='cuda:0' )
def handle_missing(features):
features['Functional'] = features['Functional'].fillna('Typ')
 features['Electrical'] = features['Electrical'].fillna("SBrkr")
 features['KitchenQual'] = features['KitchenQual'].fillna("TA")
 features['Exterior1st'] = features['Exterior1st'].fillna(features['Exterior1st'].mode() [0])
 features['Exterior2nd'] = features['Exterior2nd'].fillna(features['Exterior2nd'].mode() [0])
 features['SaleType'] = features['SaleType'].fillna(features['SaleType'].mode() [0])
 features['MSZoning'] = features.groupby('MSSubClass')['MSZoning'].transform(lambda x: x.fillna(x.mode() [0]))
features["PoolQC"] = features["PoolQC"].fillna("None")
 for col in('GarageYrBlt', 'GarageArea', 'GarageCars'):
features[col] = features[col].fillna(0)
 for col in ['GarageType', 'GarageFinish', 'GarageQual', 'GarageCond']:
features[col] = features[col].fillna('None')
 for col in('BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2'):
features[col] = features[col].fillna('None')
 features['LotFrontage'] = features.groupby('Neighborhood')['LotFrontage'].transform(lambda x: x.fillna(x.median()))
objects = []
for i in features.columns:
if features[i].dtype == object:
objects.append(i)
 features.update(features[objects].fillna('None'))
numeric_dtypes = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
numeric = []
for i in features.columns:
if features[i].dtype in numeric_dtypes:
numeric.append(i)
 features.update(features[numeric].fillna(0))
return features
all_features = handle_missing(all_features )<count_missing_values>
learn = cnn_learner(data, base_arch=models.resnet50, metrics = [quadratic_kappa], pretrained=False)
 learn.load('resnet50' )
total = all_features.isnull().sum().sort_values(ascending=False)
 percent =(all_features.isnull().sum() /all_features.isnull().count() ).sort_values(ascending=False)
 missing_data = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
 missing_data.head(25 )<define_variables>
learn_2 = cnn_learner(data, base_arch=models.resnet152, metrics = [quadratic_kappa], pretrained=False)
 learn_2.load('resnet152' )
numeric_dtypes = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
numeric = []
for i in all_features.columns:
if all_features[i].dtype in numeric_dtypes:
numeric.append(i )<concatenate>
interp = ClassificationInterpretation.from_learner(learn)
 losses,idxs = interp.top_losses()
len(data.valid_ds)==len(losses)==len(idxs )
for i in skew_index:
all_features[i] = boxcox1p(all_features[i], boxcox_normmax(all_features[i] + 1))<feature_engineering>
import numpy as np
import pandas as pd
import os
import scipy as sp
from functools import partial
from sklearn import metrics
from collections import Counter
import json
all_features['BsmtFinType1_Unf'] = 1*(all_features['BsmtFinType1'] == 'Unf')
 all_features['HasWoodDeck'] =(all_features['WoodDeckSF'] == 0)* 1
all_features['HasOpenPorch'] =(all_features['OpenPorchSF'] == 0)* 1
all_features['HasEnclosedPorch'] =(all_features['EnclosedPorch'] == 0)* 1
all_features['Has3SsnPorch'] =(all_features['3SsnPorch'] == 0)* 1
all_features['HasScreenPorch'] =(all_features['ScreenPorch'] == 0)* 1
all_features['YearsSinceRemodel'] = all_features['YrSold'].astype(int)- all_features['YearRemodAdd'].astype(int)
 all_features['Total_Home_Quality'] = all_features['OverallQual'] + all_features['OverallCond']
all_features = all_features.drop(['Utilities', 'Street', 'PoolQC',], axis=1)
 all_features['TotalSF'] = all_features['TotalBsmtSF'] + all_features['1stFlrSF'] + all_features['2ndFlrSF']
all_features['YrBltAndRemod'] = all_features['YearBuilt'] + all_features['YearRemodAdd']
all_features['Total_sqr_footage'] =(all_features['BsmtFinSF1'] + all_features['BsmtFinSF2'] +
all_features['1stFlrSF'] + all_features['2ndFlrSF'])
 all_features['Total_Bathrooms'] =(all_features['FullBath'] +(0.5 * all_features['HalfBath'])+
all_features['BsmtFullBath'] +(0.5 * all_features['BsmtHalfBath']))
all_features['Total_porch_sf'] =(all_features['OpenPorchSF'] + all_features['3SsnPorch'] +
all_features['EnclosedPorch'] + all_features['ScreenPorch'] +
all_features['WoodDeckSF'])
 all_features['TotalBsmtSF'] = all_features['TotalBsmtSF'].apply(lambda x: np.exp(6)if x <= 0.0 else x)
 all_features['2ndFlrSF'] = all_features['2ndFlrSF'].apply(lambda x: np.exp(6.5)if x <= 0.0 else x)
 all_features['GarageArea'] = all_features['GarageArea'].apply(lambda x: np.exp(6)if x <= 0.0 else x)
 all_features['GarageCars'] = all_features['GarageCars'].apply(lambda x: 0 if x <= 0.0 else x)
 all_features['LotFrontage'] = all_features['LotFrontage'].apply(lambda x: np.exp(4.2)if x <= 0.0 else x)
 all_features['MasVnrArea'] = all_features['MasVnrArea'].apply(lambda x: np.exp(4)if x <= 0.0 else x)
 all_features['BsmtFinSF1'] = all_features['BsmtFinSF1'].apply(lambda x: np.exp(6.5)if x <= 0.0 else x)
 all_features['haspool'] = all_features['PoolArea'].apply(lambda x: 1 if x > 0 else 0)
 all_features['has2ndfloor'] = all_features['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)
 all_features['hasgarage'] = all_features['GarageArea'].apply(lambda x: 1 if x > 0 else 0)
 all_features['hasbsmt'] = all_features['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 all_features['hasfireplace'] = all_features['Fireplaces'].apply(lambda x: 1 if x > 0 else 0 )<feature_engineering>
class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
ll = metrics.cohen_kappa_score(y, X_p, weights='quadratic')
 return -ll
def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X=X, y=y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 print(-loss_partial(self.coef_['x']))
def predict(self, X, coef):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
return X_p
def coefficients(self):
return self.coef_['x']
def logs(res, ls):
m = res.shape[1]
for l in ls:
res = res.assign(newcol=pd.Series(np.log(1.01+res[l])).values)
 res.columns.values[m] = l + '_log'
m += 1
return res
log_features = ['LotFrontage','LotArea','MasVnrArea','BsmtFinSF1','BsmtFinSF2','BsmtUnfSF',
'TotalBsmtSF','1stFlrSF','2ndFlrSF','LowQualFinSF','GrLivArea',
'BsmtFullBath','BsmtHalfBath','FullBath','HalfBath','BedroomAbvGr','KitchenAbvGr',
'TotRmsAbvGrd','Fireplaces','GarageCars','GarageArea','WoodDeckSF','OpenPorchSF',
'EnclosedPorch','3SsnPorch','ScreenPorch','PoolArea','MiscVal','YearRemodAdd','TotalSF']
all_features = logs(all_features, log_features )<compute_test_metric>
optR = OptimizedRounder()
def squares(res, ls):
m = res.shape[1]
for l in ls:
res = res.assign(newcol=pd.Series(res[l]*res[l] ).values)
 res.columns.values[m] = l + '_sq'
m += 1
return res
squared_features = ['YearRemodAdd', 'LotFrontage_log',
'TotalBsmtSF_log', '1stFlrSF_log', '2ndFlrSF_log', 'GrLivArea_log',
'GarageCars_log', 'GarageArea_log']
all_features = squares(all_features, squared_features )<categorify>
sample_df = pd.read_csv('.. /input/aptos2019-blindness-detection/sample_submission.csv')
 sample_df.head()
all_features = pd.get_dummies(all_features ).reset_index(drop=True)
 all_features.shape<count_duplicates>
learn.data.add_test(ImageList.from_df(sample_df,'.. /input/aptos2019-blindness-detection',folder='test_images',suffix='.png'))
all_features = all_features.loc[:,~all_features.columns.duplicated() ]<import_modules>
preds,y = learn.get_preds(ds_type=DatasetType.Test )
from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor, AdaBoostRegressor, BaggingRegressor
from sklearn.kernel_ridge import KernelRidge
from sklearn.linear_model import Ridge, RidgeCV
from sklearn.linear_model import ElasticNet, ElasticNetCV
from sklearn.svm import SVR
from mlxtend.regressor import StackingCVRegressor
import lightgbm as lgb
from lightgbm import LGBMRegressor
from xgboost import XGBRegressor
from sklearn.tree import DecisionTreeRegressor
from sklearn.ensemble import RandomForestRegressor
from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import KFold, cross_val_score
from sklearn.metrics import mean_squared_error
from sklearn.preprocessing import OneHotEncoder
from sklearn.preprocessing import LabelEncoder
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import scale
from sklearn.preprocessing import StandardScaler
from sklearn.preprocessing import RobustScaler
from sklearn.decomposition import PCA<choose_model_class>
learn_2.data.add_test(ImageList.from_df(sample_df,'.. /input/aptos2019-blindness-detection',folder='test_images',suffix='.png'))
kf = KFold(n_splits=12, random_state=42, shuffle=True )<compute_test_metric>
preds_2, y_2 = learn_2.get_preds(ds_type=DatasetType.Test )
def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))
def cv_rmse(model, X=X):
rmse = np.sqrt(-cross_val_score(model, X, train_labels, scoring="neg_mean_squared_error", cv=kf))
return(rmse )<choose_model_class>
preds_avg =(preds * 0.6 + preds_2 * 0.4)
 test_predictions = optR.predict(preds_avg, coefficients )
lightgbm = LGBMRegressor(objective='regression',
num_leaves=6,
learning_rate=0.01,
n_estimators=7000,
max_bin=200,
bagging_fraction=0.8,
bagging_freq=4,
bagging_seed=8,
feature_fraction=0.2,
feature_fraction_seed=8,
min_sum_hessian_in_leaf = 11,
verbose=-1,
random_state=42)
 xgboost = XGBRegressor(learning_rate=0.01,
n_estimators=6000,
max_depth=4,
min_child_weight=0,
gamma=0.6,
subsample=0.7,
colsample_bytree=0.7,
objective='reg:linear',
nthread=-1,
scale_pos_weight=1,
seed=27,
reg_alpha=0.00006,
random_state=42)
 ridge_alphas = [1e-15, 1e-10, 1e-8, 9e-4, 7e-4, 5e-4, 3e-4, 1e-4, 1e-3, 5e-2, 1e-2, 0.1, 0.3, 1, 3, 5, 10, 15, 18, 20, 30, 50, 75, 100]
ridge = make_pipeline(RobustScaler() , RidgeCV(alphas=ridge_alphas, cv=kf))
svr = make_pipeline(RobustScaler() , SVR(C= 20, epsilon= 0.008, gamma=0.0003))
gbr = GradientBoostingRegressor(n_estimators=6000,
learning_rate=0.01,
max_depth=4,
max_features='sqrt',
min_samples_leaf=15,
min_samples_split=10,
loss='huber',
random_state=42)
 rf = RandomForestRegressor(n_estimators=1200,
max_depth=15,
min_samples_split=5,
min_samples_leaf=5,
max_features=None,
oob_score=True,
random_state=42)
 stack_gen = StackingCVRegressor(regressors=(xgboost, lightgbm, svr, ridge, gbr, rf),
meta_regressor=xgboost,
use_features_in_secondary=True)
 dtreg = DecisionTreeRegressor(random_state = 100)
 rfr = RandomForestRegressor(n_estimators = 100, random_state = 0 )<compute_test_metric>
sample_df.diagnosis = test_predictions.astype(int)
 sample_df.head()
scores = {}
score = cv_rmse(lightgbm)
 print("lightgbm: {:.4f}({:.4f})".format(score.mean() , score.std()))
scores['lgb'] =(score.mean() , score.std() )<compute_test_metric>
sample_df.to_csv('submission.csv',index=False )
<compute_test_metric><EOS> 
sample_df.diagnosis.value_counts()