Kaggle competition: APTOS 2019 Blindness Detection
<SOS> metric: QuadraticWeightedKappa Kaggle data source: aptos-2019-blindness-detection<feature_engineering>
%reload_ext autoreload
%autoreload 2
%matplotlib inline
def calculate_abs_different(df):
df['abs_diff_longitude'] =(df.dropoff_longitude - df.pickup_longitude ).abs()
df['abs_diff_latitude'] =(df.dropoff_latitude - df.pickup_latitude ).abs()
calculate_abs_different(df_train)
 calculate_abs_different(df_test )<feature_engineering>
import fastai
from fastai import *
from fastai.vision import *
from fastai.callbacks import *
import cv2
import pandas as pd
import matplotlib.pyplot as plt
def convert_different_miles(df):
df['abs_diff_longitude'] = df.abs_diff_longitude*50
df['abs_diff_latitude'] = df.abs_diff_latitude*69
convert_different_miles(df_train)
 convert_different_miles(df_test )<feature_engineering>
print('Make sure cudnn is enabled:', torch.backends.cudnn.enabled )
meas_ang = 0.506
def add_distance(df):
df['Euclidean'] =(df.abs_diff_latitude**2 + df.abs_diff_longitude**2)**0.5
df['delta_manh_long'] =(df.Euclidean*np.sin(np.arctan(df.abs_diff_longitude / df.abs_diff_latitude)-meas_ang)).abs()
df['delta_manh_lat'] =(df.Euclidean*np.cos(np.arctan(df.abs_diff_longitude / df.abs_diff_latitude)-meas_ang)).abs()
df['distance'] = df.delta_manh_long + df.delta_manh_lat
df.drop(['abs_diff_longitude', 'abs_diff_latitude','Euclidean', 'delta_manh_long', 'delta_manh_lat'], axis=1, inplace=True)
 add_distance(df_train)
 add_distance(df_test )<prepare_x_and_y>
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
SEED = 1667
seed_everything(SEED )
def calculate_direction(pickup_lat, pickup_lon, dropoff_lat, dropoff_lon):

 R_earth = 6371
pickup_lat, pickup_lon, dropoff_lat, dropoff_lon = map(np.radians,
[pickup_lat, pickup_lon,
dropoff_lat, dropoff_lon])
 dlat = dropoff_lat - pickup_lat
dlon = pickup_lon - dropoff_lon
a = np.arctan2(np.sin(dlon * np.cos(dropoff_lat)) ,np.cos(pickup_lat)* np.sin(dropoff_lat)- np.sin(pickup_lat)* np.cos(dropoff_lat)* np.cos(dlon))
return a<feature_engineering>
base_image_dir = os.path.join('.. ', 'input/aptos2019-blindness-detection/')
 train_dir = os.path.join(base_image_dir,'train_images/')
 df = pd.read_csv(os.path.join(base_image_dir, 'train.csv'))
df['path'] = df['id_code'].map(lambda x: os.path.join(train_dir,'{}.png'.format(x)))
 df = df.drop(columns=['id_code'])
 df = df.sample(frac=1 ).reset_index(drop=True)
 df.head(10 )
train_df['direction'] = calculate_direction(train_df['pickup_latitude'].values, train_df['pickup_longitude'].values,
train_df['dropoff_latitude'].values , train_df['dropoff_longitude'].values)
 df_test['direction'] = calculate_direction(df_test['pickup_latitude'].values, df_test['pickup_longitude'].values,
df_test['dropoff_latitude'].values , df_test['dropoff_longitude'].values )<feature_engineering>
tfms = get_transforms(do_flip=True, flip_vert=True, max_rotate=0.10, max_zoom=1.3, max_warp=0.0, max_lighting=0.2)
 data =(
 src.transform(tfms,size=128)
.databunch(bs=bs)
.normalize(imagenet_stats)
 )
train_df['pickup_latitude'].apply(lambda x: np.radians(x))
train_df['pickup_longitude'].apply(lambda x: np.radians(x))
train_df['dropoff_latitude'].apply(lambda x: np.radians(x))
train_df['dropoff_longitude'].apply(lambda x: np.radians(x))
df_test['pickup_latitude'].apply(lambda x: np.radians(x))
df_test['pickup_longitude'].apply(lambda x: np.radians(x))
df_test['dropoff_latitude'].apply(lambda x: np.radians(x))
df_test['dropoff_longitude'].apply(lambda x: np.radians(x))<prepare_x_and_y>
def quadratic_kappa(y_hat, y):
return torch.tensor(cohen_kappa_score(torch.round(y_hat), y, weights='quadratic'),device='cuda:0' )
df_train.drop(columns=['pickup_datetime'], inplace=True)
 y = df_train['fare_amount']
df_train = df_train.drop(columns=['fare_amount'] )<split>
learn = cnn_learner(data, base_arch=models.densenet161 ,metrics=[quadratic_kappa],
callback_fns=[partial(EarlyStoppingCallback, monitor='quadratic_kappa',
min_delta=0.01, patience=3)],
model_dir='/kaggle',pretrained=True)
x_train,x_test,y_train,y_test = train_test_split(df_train,y,random_state=123,test_size=0.1 )<init_hyperparams>
learn.fit_one_cycle(5, 2e-2)
params = {
'boosting_type':'gbdt',
'objective': 'regression',
'nthread': 4,
'num_leaves': 31,
'learning_rate': 0.1,
'max_depth': -1,
'subsample': 0.8,
'bagging_fraction' : 1,
'max_bin' : 10000 ,
'bagging_freq': 10,
'metric': 'rmse',
'zero_as_missing': True,
'num_rounds':50000
}<train_model>
learn.unfreeze()
learn.lr_find()
learn.recorder.plot()
train_set = lgbm.Dataset(x_train, y_train, silent=False,categorical_feature=['year','month','day','day_of_week'])
 valid_set = lgbm.Dataset(x_test, y_test, silent=False,categorical_feature=['year','month','day','day_of_week'])
 model = lgbm.train(params, train_set = train_set, num_boost_round=10000,early_stopping_rounds=1000,verbose_eval=500, valid_sets=valid_set )<drop_column>
learn.fit_one_cycle(10, max_lr=slice(1e-5,1e-3))
test_key = df_test['key']
df_test.drop(columns=["pickup_datetime",'key'], axis=1, inplace=True )<predict_on_test>
valid_preds = learn.get_preds(ds_type=DatasetType.Valid )
prediction = model.predict(df_test, num_iteration = model.best_iteration)
<save_to_csv>
import numpy as np
import pandas as pd
import os
import scipy as sp
from functools import partial
from sklearn import metrics
from collections import Counter
import json
submission = pd.DataFrame({
"key": test_key,
"fare_amount": prediction
})
 submission.to_csv('taxi_fare_submission.csv',index=False )<load_from_csv>
class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
ll = metrics.cohen_kappa_score(y, X_p, weights='quadratic')
 return -ll
def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X=X, y=y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 print(-loss_partial(self.coef_['x']))
def predict(self, X, coef):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
return X_p
def coefficients(self):
return self.coef_['x']
df = pd.read_csv('/kaggle/input/new-york-city-taxi-fare-prediction/train.csv', nrows = 4000000)
 test_df = pd.read_csv('/kaggle/input/new-york-city-taxi-fare-prediction/test.csv')
 df.shape, test_df.shape<count_missing_values>
optR = OptimizedRounder()
optR.fit(valid_preds[0],valid_preds[1] )
df.isnull().sum().sort_index() /len(df )<correct_missing_values>
sample_df = pd.read_csv('.. /input/aptos2019-blindness-detection/sample_submission.csv')
 sample_df.head()
df.dropna(subset=['dropoff_latitude', 'dropoff_longitude'], inplace = True )<drop_column>
learn.data.add_test(ImageList.from_df(sample_df,'.. /input/aptos2019-blindness-detection',folder='test_images',suffix='.png'))
df.drop(df[df['fare_amount'] < 2.5].index, axis=0, inplace = True)
 df.drop(df[df['fare_amount'] > 500].index, axis=0, inplace = True )<sort_values>
preds,y = learn.TTA(ds_type=DatasetType.Test)
<drop_column><EOS> 
test_predictions = optR.predict(preds, coefficients)
 sample_df.diagnosis = test_predictions.astype(int)
 sample_df.head()
sample_df.to_csv('submission.csv',index=False )