Kaggle competition: APTOS 2019 Blindness Detection
<SOS> metric: QuadraticWeightedKappa Kaggle data source: aptos-2019-blindness-detection<predict_on_test>
%reload_ext autoreload
%autoreload 2
%matplotlib inline
y_pred_xg_boost = xg_boost.predict(X_val)
 print('XG Boost - Root Mean Squared Error: %.2f'
% math.sqrt(mean_squared_error(y_val, y_pred_xg_boost)))
<feature_engineering>
import fastai
from fastai import *
from fastai.vision import *
from fastai.callbacks import *
import cv2
import pandas as pd
import matplotlib.pyplot as plt
test_data = pd.read_csv(test_path)
 test_data['pickup_datetime'] = pd.to_datetime(test_data['pickup_datetime'],format='%Y-%m-%d %H:%M:%S %Z')
 test_data['pickup_year'] = test_data['pickup_datetime'].dt.year
test_data['pickup_quarter'] = test_data['pickup_datetime'].dt.quarter
test_data['pickup_month'] = test_data['pickup_datetime'].dt.month
test_data['pickup_day'] = test_data['pickup_datetime'].dt.day
test_data['pickup_hour'] = test_data['pickup_datetime'].dt.hour
test_data['pickup_month_cos']=np.cos(( test_data['pickup_month']-1)*(2*(np.pi/12)))
 test_data['pickup_month_sin']=np.sin(( test_data['pickup_month']-1)*(2*(np.pi/12)))
 test_data['pickup_day_cos']=np.cos(( test_data['pickup_day']-1)*(2*(np.pi/30)))
 test_data['pickup_day_sin']=np.sin(( test_data['pickup_day']-1)*(2*(np.pi/30)))
 test_data['pickup_quarter_cos']=np.cos(( test_data['pickup_quarter']-1)*(2*(np.pi/4)))
 test_data['pickup_quarter_sin']=np.sin(( test_data['pickup_quarter']-1)*(2*(np.pi/4)))
 test_data['pickup_hour_cos']=np.cos(( test_data['pickup_hour']-1)*(2*(np.pi/24)))
 test_data['pickup_hour_sin']=np.sin(( test_data['pickup_hour']-1)*(2*(np.pi/24)))
 test_data['pickup_year_age']=2020-test_data['pickup_year']
test_data.drop(['pickup_year','pickup_quarter','pickup_month','pickup_day','pickup_hour','pickup_datetime'], axis=1, inplace=True)
 test_data['distance'] = distance(test_data['pickup_latitude'], test_data['pickup_longitude'], test_data['dropoff_latitude'], test_data['dropoff_longitude'])
 test_data.head()<predict_on_test>
print('Make sure cudnn is enabled:', torch.backends.cudnn.enabled )
y_predictions = xg_boost.predict(test_data.iloc[:, 1:] )<save_to_csv>
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
SEED = 1667
seed_everything(SEED )
submission = pd.DataFrame({'key': test_data['key'], 'fare_amount': y_predictions},columns = ['key', 'fare_amount'])
 submission.to_csv('submission.csv', index = False )<import_modules>
def get_2015_df() :
base_image_dir = os.path.join('.. ', 'input/resized-2015-2019-blindness-detection-images/')
 train_dir = os.path.join(base_image_dir,'resized train 15/')
 df = pd.read_csv(os.path.join(base_image_dir, 'labels/trainLabels15.csv'))
df.columns = ['image', 'diagnosis']
df['path'] = df['image'].map(lambda x: os.path.join(train_dir,'{}.jpg'.format(x)))
 df = df.drop(columns=['image'])
 df = df.sample(frac=1 ).reset_index(drop=True)
 return df
from pandas import read_csv, DataFrame, to_datetime
from numpy import radians, sin, cos, arcsin, sqrt, mean
from sklearn import ensemble
import time
from datetime import datetime
import xgboost as xgb
from sklearn.model_selection import train_test_split 
<load_from_csv>
def get_df_2019() :
base_image_dir = os.path.join('.. ', 'input/resized-2015-2019-blindness-detection-images/')
 train_dir = os.path.join(base_image_dir,'resized train 19/')
 df = pd.read_csv(os.path.join(base_image_dir, 'labels/trainLabels19.csv'))
df['path'] = df['id_code'].map(lambda x: os.path.join(train_dir,'{}.jpg'.format(x)))
 df = df.drop(columns=['id_code'])
 df = df.sample(frac=1 ).reset_index(drop=True)
 test_df = pd.read_csv('.. /input/aptos2019-blindness-detection/sample_submission.csv')
 return df, test_df
train = read_csv(".. /input/train.csv")
 test = read_csv(".. /input/test.csv")
<define_variables>
base_image_dir = os.path.join('.. ', 'input/aptos2019-blindness-detection/')
 train_dir = os.path.join(base_image_dir,'train_images/')
 df = pd.read_csv(os.path.join(base_image_dir, 'train.csv'))
df['path'] = df['id_code'].map(lambda x: os.path.join(train_dir,'{}.png'.format(x)))
 df = df.drop(columns=['id_code'])
 df = df.sample(frac=1 ).reset_index(drop=True)
 df.head(10 )
train = train[:5000000]<count_missing_values>
tfms = get_transforms(do_flip=True, flip_vert=True, max_rotate=0.10, max_zoom=1.3, max_warp=0.0, max_lighting=0.2)
 data =(
 src.transform(tfms,size=128)
.databunch() 
.normalize(imagenet_stats)
 )
print(train.isnull().sum() )<drop_column>
def quadratic_kappa(y_hat, y):
return torch.tensor(cohen_kappa_score(torch.round(y_hat), y, weights='quadratic'),device='cuda:0' )
train = train.dropna(how = 'any', axis = 'rows' )<categorify>
learn = cnn_learner(data, base_arch=models.resnet50 ,metrics=[quadratic_kappa],model_dir='/kaggle',pretrained=True )
pickup_longitude_min = test.pickup_longitude.min()
pickup_longitude_max = test.pickup_latitude.max()
pickup_latitude_min = test.pickup_latitude.min()
pickup_latitude_max = test.pickup_latitude.max()
dropoff_longitude_min = test.dropoff_longitude.min()
dropoff_longitude_max = test.dropoff_longitude.max()
dropoff_latitude_min = test.dropoff_latitude.min()
dropoff_latitude_max = test.dropoff_latitude.max()
train = train.loc[(train['fare_amount'] > 0)&(train['fare_amount'] < 300)]
train = train.loc[(train['pickup_longitude'] > pickup_longitude_min)&(train['pickup_longitude'] < pickup_longitude_max)]
train = train.loc[(train['pickup_latitude'] > pickup_latitude_min)&(train['pickup_latitude'] < pickup_latitude_max)]
train = train.loc[(train['dropoff_longitude'] > dropoff_longitude_min)&(train['dropoff_longitude'] < dropoff_longitude_max)]
train = train.loc[(train['dropoff_latitude'] > dropoff_latitude_min)&(train['dropoff_latitude'] < dropoff_latitude_max)]<compute_test_metric>
learn.fit_one_cycle(3, 1e-2 )
def rasst(value1, value2, value3, value4):
longitude_1, latitude_1, longitude_2, latitude_2 = value1, value2, value3, value4
longitude_1, latitude_1, longitude_2, latitude_2 = map(radians, [longitude_1, latitude_1, longitude_2, latitude_2])
 dlongitude = longitude_2 - longitude_1
dlatitude = latitude_2 - latitude_1
value = sin(dlatitude/2.0)**2 + cos(latitude_1)* cos(latitude_2)* sin(dlongitude/2.0)**2
c = 2 * arcsin(sqrt(value))
km = c * 6367
return km<data_type_conversions>
learn.data = data =(
 src.transform(tfms,size=256)
.databunch() 
.normalize(imagenet_stats)
)
 learn.lr_find()
learn.recorder.plot()
train['pickup_datetime'] = to_datetime(train['pickup_datetime'])
 train['hour_of_day'] = train.pickup_datetime.dt.hour.astype(float)
 train['day'] = train.pickup_datetime.dt.day.astype(float)
 train['week'] = train.pickup_datetime.dt.week.astype(float)
 train['month'] = train.pickup_datetime.dt.month.astype(float)
 train['day_of_year'] = train.pickup_datetime.dt.dayofyear.astype(float)
 train['week_of_year'] = train.pickup_datetime.dt.weekofyear.astype(float)
 train['passenger_count'] = train['passenger_count'].astype(float)
 train['rasst'] = rasst(train['pickup_longitude'], train['pickup_latitude'], train['dropoff_longitude'], train['dropoff_latitude'])
 test['pickup_datetime'] = to_datetime(test['pickup_datetime'])
 test['hour_of_day'] = test.pickup_datetime.dt.hour.astype(float)
 test['day'] = test.pickup_datetime.dt.day.astype(float)
 test['week'] = test.pickup_datetime.dt.week.astype(float)
 test['month'] = test.pickup_datetime.dt.month.astype(float)
 test['day_of_year'] = test.pickup_datetime.dt.dayofyear.astype(float)
 test['week_of_year'] = test.pickup_datetime.dt.weekofyear.astype(float)
 test['passenger_count'] = test['passenger_count'].astype(float)
 test['rasst'] = rasst(test['pickup_longitude'], test['pickup_latitude'], test['dropoff_longitude'], test['dropoff_latitude'] )<prepare_x_and_y>
lr = 1e-2
learn.fit_one_cycle(5, lr )
train_y = train["fare_amount"]
test_key = test['key']
train_x = train.drop(["fare_amount", "key"], axis = 1)
 train_x = train_x.drop(['pickup_datetime'], axis = 1)
 test = test.drop(['pickup_datetime', 'key'], axis = 1)
<prepare_x_and_y>
learn.unfreeze()
learn.lr_find()
learn.recorder.plot()
train_xgb = xgb.DMatrix(train_x, train_y)
 test_xgb = xgb.DMatrix(test)
<predict_on_test>
learn.fit_one_cycle(10, slice(1e-6,1e-4))
num_round = 10
param = {'max_depth':20, 'eta':0.5,'min_child-weight':2, 'gamma':2, 'booster':'dart', 'three-method':'approx', 'normalize_type':'forest', 'rate_drop':0.5, 'eval_metric':'rmse'}
train = xgb.train(param, train_xgb, num_round)
 predict = train.predict(test_xgb, ntree_limit = num_round )<save_to_csv>
learn.export()
learn.save('resnet_old_image_weight' )
submission = DataFrame({
"key": test_key,
"fare_amount": predict.round(2)
 })
 submission.to_csv('taxi_fare_submission.csv',index=False)
 submission.head()<load_from_csv>
interp = ClassificationInterpretation.from_learner(learn)
 losses,idxs = interp.top_losses()
len(data.valid_ds)==len(losses)==len(idxs )
train = pd.read_csv(".. /input/train.csv", nrows = 1000000)
 test = pd.read_csv(".. /input/test.csv" )<sort_values>
valid_preds = learn.get_preds(ds_type=DatasetType.Valid )
train.isnull().sum().sort_values(ascending=False )<drop_column>
import numpy as np
import pandas as pd
import os
import scipy as sp
from functools import partial
from sklearn import metrics
from collections import Counter
import json
train = train.drop(train[train.isnull().any(1)].index, axis = 0 )<count_values>
class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
ll = metrics.cohen_kappa_score(y, X_p, weights='quadratic')
 return -ll
def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X=X, y=y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 print(-loss_partial(self.coef_['x']))
def predict(self, X, coef):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
return X_p
def coefficients(self):
return self.coef_['x']
Counter(train['fare_amount']<0 )<drop_column>
optR = OptimizedRounder()
optR.fit(valid_preds[0],valid_preds[1] )
train = train.drop(train[train['fare_amount']<0].index, axis=0)
 train.shape<sort_values>
sample_df = pd.read_csv('.. /input/aptos2019-blindness-detection/sample_submission.csv')
 sample_df.head()
<define_variables><EOS> 
learn.data.add_test(ImageList.from_df(sample_df,'.. /input/aptos2019-blindness-detection',folder='test_images',suffix='.png'))
preds,y = learn.get_preds(ds_type=DatasetType.Test)
 test_predictions = optR.predict(preds, coefficients)
 sample_df.diagnosis = test_predictions.astype(int)
 sample_df.head()
sample_df.to_csv('submission.csv',index=False )