Kaggle competition: APTOS 2019 Blindness Detection
<SOS> metric: QuadraticWeightedKappa Kaggle data source: aptos-2019-blindness-detection<define_variables>
%reload_ext autoreload
%autoreload 2
%matplotlib inline
train = train.loc[train.pickup_longitude < -72.5]
train = train.loc[train.pickup_longitude > -74.5]<filter>
import fastai
from fastai import *
from fastai.vision import *
from fastai.callbacks import *
import cv2
import pandas as pd
import matplotlib.pyplot as plt
train = train.loc[train.pickup_latitude < 42]
train = train.loc[train.pickup_latitude > 40]<filter>
print('Make sure cudnn is enabled:', torch.backends.cudnn.enabled )
train = train.loc[train.dropoff_longitude < -72.5]
train = train.loc[train.dropoff_longitude > -74.5]<filter>
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
SEED = 1667
seed_everything(SEED )
train = train.loc[train.dropoff_latitude < 42]
train = train.loc[train.dropoff_latitude > 40]<feature_engineering>
base_image_dir = os.path.join('.. ', 'input/aptos2019-blindness-detection/')
 train_dir = os.path.join(base_image_dir,'train_images/')
 df = pd.read_csv(os.path.join(base_image_dir, 'train.csv'))
df['path'] = df['id_code'].map(lambda x: os.path.join(train_dir,'{}.png'.format(x)))
 df = df.drop(columns=['id_code'])
 df = df.sample(frac=1 ).reset_index(drop=True)
 df.head(10 )
train['longitude_diff'] = train['dropoff_longitude'] - train['pickup_longitude']
train['latitude_diff'] = train['dropoff_latitude'] - train['pickup_latitude']<filter>
src =(ImageList.from_df(df=df,path='./',cols='path')
.split_by_rand_pct(0.2, seed=42)
.label_from_df(cols='diagnosis',label_cls=FloatList)
)
 src
train = train.loc[train.longitude_diff > -0.9]
train = train.loc[train.longitude_diff < 0.5]
train = train.loc[train.latitude_diff > -0.7]
train = train.loc[train.latitude_diff < 0.3]<filter>
tfms = get_transforms(do_flip=True, flip_vert=True, max_rotate=0.10, max_zoom=1.3, max_warp=0.0, max_lighting=0.2)
 data =(
 src.transform(tfms,size=224)
.databunch() 
.normalize(imagenet_stats)
)
 data
train = train.loc[train.passenger_count > 0]
train = train.loc[train.passenger_count <= 6]<feature_engineering>
def quadratic_kappa(y_hat, y):
return torch.tensor(cohen_kappa_score(torch.round(y_hat), y, weights='quadratic'),device='cuda:0' )
train['year'] = train.pickup_datetime.apply(lambda x: x[:4])
 test['year'] = test.pickup_datetime.apply(lambda x: x[:4] )<feature_engineering>
kappa = KappaScore()
kappa.weights = "quadratic"
train['month'] = train.pickup_datetime.apply(lambda x: x[5:7])
 test['month'] = test.pickup_datetime.apply(lambda x: x[5:7] )<feature_engineering>
learn = cnn_learner(data, base_arch=models.resnet152 ,
metrics=[quadratic_kappa],model_dir='/kaggle',
pretrained=True,
callback_fns=[partial(EarlyStoppingCallback, monitor='quadratic_kappa',
min_delta=0.01, patience=3)] )
train['hour'] = train.pickup_datetime.apply(lambda x: x[11:13])
 test['hour'] = test.pickup_datetime.apply(lambda x: x[11:13] )<data_type_conversions>
learn.fit_one_cycle(6, 3e-2)
train['pickup_datetime'] = train.pickup_datetime.apply(
 lambda x: datetime.datetime.strptime(x[:10], '%Y-%m-%d'))
test['pickup_datetime'] = test.pickup_datetime.apply(
 lambda x: datetime.datetime.strptime(x[:10], '%Y-%m-%d'))<feature_engineering>
learn.save('stage1' )
train['day_of_week'] = train.pickup_datetime.apply(lambda x: x.weekday())
 test['day_of_week'] = test.pickup_datetime.apply(lambda x: x.weekday() )<feature_engineering>
learn.fit_one_cycle(10, max_lr=slice(1e-6,1e-3))
train['pickup_date'] = train.pickup_datetime.apply(lambda x: x.date())
 test['pickup_date'] = test.pickup_datetime.apply(lambda x: x.date() )<data_type_conversions>
valid_preds = learn.get_preds(ds_type=DatasetType.Valid )
cal = USFederalHolidayCalendar()
holidays = cal.holidays(start='2009-01-01', end='2015-12-31' ).to_pydatetime()
train['holidat_or_not'] = train.pickup_datetime.apply(lambda x: 1 if x in holidays else 0)
 test['holidat_or_not'] = test.pickup_datetime.apply(lambda x: 1 if x in holidays else 0 )<drop_column>
import numpy as np
import pandas as pd
import os
import scipy as sp
from functools import partial
from sklearn import metrics
from collections import Counter
import json
train = train.drop(['key','pickup_datetime','pickup_date'],axis=1)
 test = test.drop(['key','pickup_datetime','pickup_date'],axis=1 )<data_type_conversions>
class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
ll = metrics.cohen_kappa_score(y, X_p, weights='quadratic')
 return -ll
def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X=X, y=y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 print(-loss_partial(self.coef_['x']))
def predict(self, X, coef):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
return X_p
def coefficients(self):
return self.coef_['x']
train['year'] = train['year'].astype('int')
 train['hour'] = train['hour'].astype('int')
 test['year'] = test['year'].astype('int')
 test['hour'] = test['hour'].astype('int' )<import_modules>
optR = OptimizedRounder()
optR.fit(valid_preds[0],valid_preds[1] )
from sklearn.preprocessing import StandardScaler<define_variables>
sample_df = pd.read_csv('.. /input/aptos2019-blindness-detection/sample_submission.csv')
 sample_df.head()
sample = train[:50000][['pickup_longitude','pickup_latitude']]<normalization>
learn.data.add_test(ImageList.from_df(sample_df,'.. /input/aptos2019-blindness-detection',folder='test_images',suffix='.png'))
preds,y = learn.get_preds(ds_type=DatasetType.Test)
 test_predictions = optR.predict(preds, coefficients)
 sample_df.diagnosis = test_predictions.astype(int)
 sample_df.head()
sample_df.to_csv('submission.csv',index=False )