Kaggle competition: APTOS 2019 Blindness Detection
<SOS> metric: QuadraticWeightedKappa Kaggle data source: aptos-2019-blindness-detection<data_type_conversions>
%matplotlib inline
warnings.filterwarnings('always')
 warnings.filterwarnings('ignore')
 print(os.listdir(".. /input"))
df['key'] = pd.to_datetime(df['key'])
 key = test_df.key
test_df['key'] = pd.to_datetime(test_df['key'])
 df['pickup_datetime'] = pd.to_datetime(df['pickup_datetime'])
 test_df['pickup_datetime'] = pd.to_datetime(test_df['pickup_datetime'] )<feature_engineering>
df_train = pd.read_csv('.. /input/aptos2019-blindness-detection/train.csv')
 df_test = pd.read_csv('.. /input/aptos2019-blindness-detection/test.csv')
 x_train = df_train['id_code']
y_train = df_train['diagnosis']
df['Year'] = df['pickup_datetime'].dt.year
df['Month'] = df['pickup_datetime'].dt.month
df['Date'] = df['pickup_datetime'].dt.day
df['Day of Week'] = df['pickup_datetime'].dt.dayofweek
df['Hour'] = df['pickup_datetime'].dt.hour
df.drop('pickup_datetime', axis = 1, inplace = True)
 df.drop('key', axis = 1, inplace = True)
 test_df['Year'] = test_df['pickup_datetime'].dt.year
test_df['Month'] = test_df['pickup_datetime'].dt.month
test_df['Date'] = test_df['pickup_datetime'].dt.day
test_df['Day of Week'] = test_df['pickup_datetime'].dt.dayofweek
test_df['Hour'] = test_df['pickup_datetime'].dt.hour
test_df.drop('pickup_datetime', axis = 1, inplace = True)
 test_df.drop('key', axis = 1, inplace = True )<drop_column>
import torch
import torch.utils.data
import torchvision
df.dropna(inplace=True)
 df.drop(df.index[(df.pickup_longitude < -75)| 
(df.pickup_longitude > -72)| 
(df.pickup_latitude < 40)| 
(df.pickup_latitude > 42)],inplace=True)
 df.drop(df.index[(df.dropoff_longitude < -75)| 
(df.dropoff_longitude > -72)| 
(df.dropoff_latitude < 40)| 
(df.dropoff_latitude > 42)],inplace=True )<data_type_conversions>
def get_label(diagnosis):
return ','.join([str(i)for i in range(diagnosis + 1)] )
def optimize_floats(df):
floats = df.select_dtypes(include=['float64'] ).columns.tolist()
df[floats] = df[floats].apply(pd.to_numeric, downcast='float')
 return df
def optimize_ints(df):
ints = df.select_dtypes(include=['int64'] ).columns.tolist()
df[ints] = df[ints].apply(pd.to_numeric, downcast='integer')
 return df
def optimize(df):
return optimize_floats(optimize_ints(df))<find_best_params>
df_train['label'] = df_train.diagnosis.apply(get_label )
df = optimize(df)
 test_df = optimize(test_df )<feature_engineering>
df_train.head(10)
 torch.cuda.manual_seed_all(13 )
def calc_dists(df):
df['geodesic'] = df.apply(lambda x: geodesic_dist(x), axis = 1)
 df['circle'] = df.apply(lambda x: circle_dist(x), axis = 1)
 df['jfk'] = df.apply(lambda x: jfk_dist(x), axis = 1)
 df['lga'] = df.apply(lambda x: lga_dist(x), axis = 1)
 df['ewr'] = df.apply(lambda x: ewr_dist(x), axis = 1)
 df['tsq'] = df.apply(lambda x: tsq_dist(x), axis = 1)
 df['cpk'] = df.apply(lambda x: cpk_dist(x), axis = 1)
 df['lib'] = df.apply(lambda x: lib_dist(x), axis = 1)
 df['gct'] = df.apply(lambda x: gct_dist(x), axis = 1)
 df['met'] = df.apply(lambda x: met_dist(x), axis = 1)
 df['wtc'] = df.apply(lambda x: wtc_dist(x), axis = 1)
 return df<normalization>
tfms =([RandTransform(tfm=TfmCrop(crop_pad), kwargs={'row_pct':(0.4, 1), 'col_pct':(0.1, 0.9), 'padding_mode': 'reflection'}, p=1.0, resolved={}, do_run=True, is_random=True, use_on_y=True),
RandTransform(tfm=TfmPixel(rgb_randomize), kwargs={'channel':2, 'thresh':0.1}, p=0.75, resolved={}, do_run=True, is_random=True, use_on_y=True),
RandTransform(tfm=TfmPixel(rgb_randomize), kwargs={'channel':1, 'thresh':0.1}, p=0.75, resolved={}, do_run=True, is_random=True, use_on_y=True),
RandTransform(tfm=TfmAffine(dihedral_affine), kwargs={}, p=1.0, resolved={}, do_run=True, is_random=True, use_on_y=True),
RandTransform(tfm=TfmAffine(rotate), kwargs={'degrees':(-15.0, 15.0)}, p=0.75, resolved={}, do_run=True, is_random=True, use_on_y=True),
RandTransform(tfm=TfmAffine(zoom), kwargs={'scale':(1.0, 1.05), 'row_pct':(0, 1), 'col_pct':(0, 1)}, p=0.75, resolved={}, do_run=True, is_random=True, use_on_y=True),
RandTransform(tfm=TfmLighting(brightness), kwargs={'change':(0.4, 0.6)}, p=0.75, resolved={}, do_run=True, is_random=True, use_on_y=True),
RandTransform(tfm=TfmLighting(contrast), kwargs={'scale':(0.8, 1.25)}, p=0.75, resolved={}, do_run=True, is_random=True, use_on_y=True)],
[RandTransform(tfm=TfmCrop(crop_pad), kwargs={}, p=1.0, resolved={}, do_run=True, is_random=True, use_on_y=True)] )
df = calc_dists(df)
 test_df = calc_dists(test_df )<find_best_params>
data = ImageDataBunch.from_df('./',
df=df_train,
valid_pct=0.25,
folder=".. /input/aptos2019-blindness-detection/train_images",
suffix=".png",
ds_tfms=tfms,
size=224,
bs=156,
num_workers=32,
label_col='label', label_delim=',' ).normalize(imagenet_stats )
df = optimize(df)
 test_df = optimize(test_df )<split>
print(f'Classes:
{data.classes}' )
X, y = df.drop('fare_amount', axis = 1), df['fare_amount']
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=12 )<train_model>
def get_preds(arr):
mask = arr == 0
return np.clip(np.where(mask.any(1), mask.argmax(1), 5)- 1, 0, 4 )
dtrain = xgb.DMatrix(X_train, label=y_train)
 dvalid = xgb.DMatrix(X_test, label=y_test)
 dtest = xgb.DMatrix(test_df)
 watchlist = [(dtrain, 'train'),(dvalid, 'valid')]
xgb_params = {
'min_child_weight': 1,
'learning_rate': 0.05,
'colsample_bytree': 0.7,
'max_depth': 10,
'subsample': 0.7,
'n_estimators': 5000,
'n_jobs': -1,
'booster' : 'gbtree',
'silent': 1,
'eval_metric': 'rmse'}
model = xgb.train(xgb_params, dtrain, 700, watchlist, early_stopping_rounds=100, maximize=False, verbose_eval=50 )<compute_test_metric>
last_output = torch.tensor([
[1.7226, 1.7226, 1.7226, 1.7226, 1.7226],
[0, 0, 0, 0, 1.7226],
[0.12841, -7.6266, -6.3899, -2.1333, -0.48995],
[0.68119, 1.7226, -1.9895, -0.097746, 0.53576]
])
 arr =(torch.sigmoid(last_output)> 0.5 ).numpy() ; arr
y_train_pred = model.predict(dtrain)
 y_pred = model.predict(dvalid)
 print('Train r2 score: ', r2_score(y_train_pred, y_train))
print('Test r2 score: ', r2_score(y_test, y_pred))
train_rmse = np.sqrt(mean_squared_error(y_train_pred, y_train))
test_rmse = np.sqrt(mean_squared_error(y_test, y_pred))
print(f'Train RMSE: {train_rmse:.4f}')
 print(f'Test RMSE: {test_rmse:.4f}' )<predict_on_test>
assert(get_preds(arr)== np.array([4, 0, 0, 1])).all()
test_preds = model.predict(dtest )<save_to_csv>
class ConfusionMatrix(Callback):
"Computes the confusion matrix."
def on_train_begin(self, **kwargs):
self.n_classes = 0
def on_epoch_begin(self, **kwargs):
self.cm = None
def on_batch_end(self, last_output:Tensor, last_target:Tensor, **kwargs):
preds = torch.tensor(get_preds(( torch.sigmoid(last_output)> 0.5 ).cpu().numpy()))
targs = torch.tensor(get_preds(last_target.cpu().numpy()))
if self.n_classes == 0:
self.n_classes = last_output.shape[-1]
self.x = torch.arange(0, self.n_classes)
 cm =(( preds==self.x[:, None])&(targs==self.x[:, None, None])).sum(dim=2, dtype=torch.float32)
 if self.cm is None: self.cm = cm
else: self.cm += cm
def on_epoch_end(self, **kwargs):
self.metric = self.cm
@dataclass
class KappaScore(ConfusionMatrix):
"Compute the rate of agreement(Cohens Kappa )."
weights:Optional[str]=None
def on_epoch_end(self, last_metrics, **kwargs):
sum0 = self.cm.sum(dim=0)
 sum1 = self.cm.sum(dim=1)
 expected = torch.einsum('i,j->ij',(sum0, sum1)) / sum0.sum()
if self.weights is None:
w = torch.ones(( self.n_classes, self.n_classes))
w[self.x, self.x] = 0
elif self.weights == "linear" or self.weights == "quadratic":
w = torch.zeros(( self.n_classes, self.n_classes))
w += torch.arange(self.n_classes, dtype=torch.float)
 w = torch.abs(w - torch.t(w)) if self.weights == "linear" else(w - torch.t(w)) ** 2
else: raise ValueError('Unknown weights.Expected None, "linear", or "quadratic".')
 k = torch.sum(w * self.cm)/ torch.sum(w * expected)
 return add_metrics(last_metrics, 1-k )
test_preds = model.predict(dtest)
 submission = pd.DataFrame(
 {'key': key, 'fare_amount': test_preds},
columns = ['key', 'fare_amount'])
 submission.to_csv('submission1.csv', index = False )<load_from_csv>
class Ranger(Optimizer):
def __init__(self, params, lr=1e-2, alpha=0.5, k=8, betas= (.9,0.999), eps=1e-8, weight_decay=0.1):
if not 0.0 <= alpha <= 1.0:
raise ValueError(f'Invalid slow update rate: {alpha}')
 if not 1 <= k:
raise ValueError(f'Invalid lookahead steps: {k}')
 if not lr > 0:
raise ValueError(f'Invalid Learning Rate: {lr}')
 if not eps > 0:
raise ValueError(f'Invalid eps: {eps}')
 defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay)
 super().__init__(params,defaults)
 for group in self.param_groups:
group["step_counter"] = 0
self.alpha = alpha
self.k = k
self.radam_buffer = [[None,None,None] for ind in range(10)]
self.slow_weights = [[p.clone().detach() for p in group['params']]
for group in self.param_groups]
for w in it.chain(*self.slow_weights):
w.requires_grad = False
def __setstate__(self, state):
print("set state called")
 super(Ranger, self ).__setstate__(state)
 def step(self, closure=None):
loss = None
for group in self.param_groups:
for p in group['params']:
if p.grad is None:
continue
grad = p.grad.data.float()
if grad.is_sparse:
raise RuntimeError('RAdam does not support sparse gradients')
 p_data_fp32 = p.data.float()
state = self.state[p]
if len(state)== 0:
state['step'] = 0
state['exp_avg'] = torch.zeros_like(p_data_fp32)
 state['exp_avg_sq'] = torch.zeros_like(p_data_fp32)
 else:
state['exp_avg'] = state['exp_avg'].type_as(p_data_fp32)
 state['exp_avg_sq'] = state['exp_avg_sq'].type_as(p_data_fp32)
 exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq']
beta1, beta2 = group['betas']
exp_avg_sq.mul_(beta2 ).addcmul_(1 - beta2, grad, grad)
 exp_avg.mul_(beta1 ).add_(1 - beta1, grad)
 state['step'] += 1
buffered = self.radam_buffer[int(state['step'] % 10)]
if state['step'] == buffered[0]:
N_sma, step_size = buffered[1], buffered[2]
else:
buffered[0] = state['step']
beta2_t = beta2 ** state['step']
N_sma_max = 2 /(1 - beta2)- 1
N_sma = N_sma_max - 2 * state['step'] * beta2_t /(1 - beta2_t)
 buffered[1] = N_sma
if N_sma > 5:
step_size = group['lr'] * math.sqrt(( 1 - beta2_t)*(N_sma - 4)/(N_sma_max - 4)*(N_sma - 2)/ N_sma * N_sma_max /(N_sma_max - 2)) /(1 - beta1 ** state['step'])
 else:
step_size = group['lr'] /(1 - beta1 ** state['step'])
 buffered[2] = step_size
if group['weight_decay'] != 0:
p_data_fp32.add_(-group['weight_decay'] * group['lr'], p_data_fp32)
 if N_sma > 5:
denom = exp_avg_sq.sqrt().add_(group['eps'])
 p_data_fp32.addcdiv_(-step_size, exp_avg, denom)
 else:
p_data_fp32.add_(-step_size, exp_avg)
 p.data.copy_(p_data_fp32)
 for group,slow_weights in zip(self.param_groups,self.slow_weights):
group['step_counter'] += 1
if group['step_counter'] % self.k != 0:
continue
for p,q in zip(group['params'],slow_weights):
if p.grad is None:
continue
q.data.add_(self.alpha,p.data - q.data)
 p.data.copy_(q.data)
 return loss
optar = partial(Ranger )
df=pd.read_csv('/kaggle/input/new-york-city-taxi-fare-prediction/train.csv',parse_dates=['pickup_datetime'], nrows=500000)
 df.dropna(inplace=True )<load_from_csv>
kappa = KappaScore(weights="quadratic")
 learn = cnn_learner(data, models.resnet50, metrics=[kappa, accuracy_thresh],
opt_func = optar,
callback_fns = [
partial(EarlyStoppingCallback, monitor='kappa_score', min_delta=0.001, patience=3),
partial(ReduceLROnPlateauCallback),
partial(SaveModelCallback, every = 'improvement', monitor='kappa_score', name='bestordinal')],
model_dir="/tmp" ).to_fp16().mixup(stack_y=False )
test=pd.read_csv('/kaggle/input/new-york-city-taxi-fare-prediction/test.csv',parse_dates=['pickup_datetime'])
 test<define_variables>
lr = min_loss_lr
learn.fit_one_cycle(10, lr)
 torch.cuda.manual_seed_all(18 )
nyc_min_longitude = -74.3
nyc_max_longitude = -72
nyc_min_latitude = 40.63
nyc_max_latitude = 42 
<filter>
learn.unfreeze()
learn.lr_find
lrs = learn.recorder.lrs
losses = learn.recorder.losses
mg =(np.gradient(np.array(losses)) ).argmin()
ml = np.argmin(losses[1:])
 min_grad_lr = lrs[mg]
print(min_grad_lr)
 min_loss_lr = lrs[ml]/10
print(min_loss_lr)
 lr2 = min_loss_lr
learn.unfreeze()
learn.fit_one_cycle(10, max_lr = lr2 )
for long in ['pickup_longitude', 'dropoff_longitude']:
df = df[(df[long] > nyc_min_longitude)&(df[long] < nyc_max_longitude)]
for lat in ['pickup_latitude', 'dropoff_latitude']:
df = df[(df[lat] > nyc_min_latitude)&(df[lat] < nyc_max_latitude)]<feature_engineering>
learn.load('bestordinal' )
df.loc[df['passenger_count'] == 0, 'passenger_count'] = 1<filter>
learn.freeze()
learn.fit_one_cycle(15, max_lr=lr2/50,wd=1e-1 )
df = df[(df['fare_amount'] >0)&(df['fare_amount'] <= 100)]<compute_test_metric>
learn.load('bestordinal')
 sample_df = pd.read_csv('.. /input/aptos2019-blindness-detection/sample_submission.csv')
 sample_df.head()
def euc_distance(lat1, long1, lat2, long2):
return(((lat1-lat2)**2 +(long1-long2)**2)**0.5)
 df['travel_distance'] = euc_distance(df['pickup_latitude'], df['pickup_longitude'], df['dropoff_latitude'], df['dropoff_longitude'])
 test['travel_distance'] = euc_distance(test['pickup_latitude'], test['pickup_longitude'], test['dropoff_latitude'], test['dropoff_longitude'] )<feature_engineering>
learn.data.add_test(ImageList.from_df(sample_df,'.. /input/aptos2019-blindness-detection',folder='test_images',suffix='.png'))
df['year'] = df['pickup_datetime'].dt.year
df['month'] = df['pickup_datetime'].dt.month
df['day'] = df['pickup_datetime'].dt.day
df['day_of_week'] = df['pickup_datetime'].dt.dayofweek
df['hour'] = df['pickup_datetime'].dt.hour
df = df.drop(['pickup_datetime'], axis=1 )<feature_engineering>
preds, y = learn.get_preds(DatasetType.Test )
test['year'] = test['pickup_datetime'].dt.year
test['month'] =test['pickup_datetime'].dt.month
test['day'] = test['pickup_datetime'].dt.day
test['day_of_week'] =test['pickup_datetime'].dt.dayofweek
test['hour'] = test['pickup_datetime'].dt.hour
test = test.drop(['pickup_datetime'], axis=1 )<drop_column>
sample_df.diagnosis = get_preds(( preds > 0.5 ).cpu().numpy())
 sample_df.diagnosis.value_counts()
<count_missing_values><EOS> 
sample_df.to_csv('submission.csv',index=False )