Kaggle competition: APTOS 2019 Blindness Detection
<SOS> metric: QuadraticWeightedKappa Kaggle data source: aptos-2019-blindness-detection<drop_column>
TTA = False
del sale
del median
del sale_with_negative_price
gc.collect()<merge>
%reload_ext autoreload
%autoreload 2
%matplotlib inline
%matplotlib inline
sales_temp = sales[sales['item_cnt_day'] > 500]
print('Sold item outliers')
 items[items['item_id'].isin(sales_temp['item_id'].values)].merge(sales_temp[['item_id', 'item_cnt_day', 'date_block_num']], on = 'item_id' )<set_options>
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(42 )
del sales_temp
gc.collect()<count_duplicates>
def get_df() :
base_image_dir = os.path.join('.. ', 'input/aptos2019-blindness-detection/')
 train_dir = os.path.join(base_image_dir,'train_images/')
 df = pd.read_csv(os.path.join(base_image_dir, 'train.csv'))
df['path'] = df['id_code'].map(lambda x: os.path.join(train_dir,'{}.png'.format(x)))
 df = df.drop(columns=['id_code'])
 df = df.sample(frac=1 ).reset_index(drop=True)
 test_df = pd.read_csv('.. /input/aptos2019-blindness-detection/sample_submission.csv')
 return df, test_df
df, test_df = get_df()
print('Number of duplicates:', len(sales[sales.duplicated() ]))<count_duplicates>
def qk(y_pred, y):
return torch.tensor(cohen_kappa_score(torch.round(y_pred), y, weights='quadratic'), device='cuda:0' )
sales = sales.drop_duplicates(keep = 'first')
 print('Number of duplicates:', len(sales[sales.duplicated() ]))<feature_engineering>
class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
ll = metrics.cohen_kappa_score(y, X_p, weights='quadratic')
 return -ll
def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X=X, y=y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 print(-loss_partial(self.coef_['x']))
def predict(self, X, coef):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
return X_p
def coefficients(self):
return self.coef_['x']
start = time.time()
sales.date = sales.date.apply(lambda x: datetime.datetime.strptime(x, '%d.%m.%Y'))
print('First sale took place: ', sales.date.min())
 print('Last sale took place: ', sales.date.max())
 print('It tooks: ', round(time.time() - start), 'seconds' )<feature_engineering>
def run_subm(learn, rounder_coefficients=[0.5, 1.5, 2.5, 3.5], tta=True):
rounder = OptimizedRounder()
if tta:
preds,y = learn.TTA(ds_type=DatasetType.Test)
 else:
preds, y = learn.get_preds(DatasetType.Test)
 tst_pred = rounder.predict(preds, rounder_coefficients)
 test_df.diagnosis = tst_pred.astype(int)
 test_df.to_csv('submission.csv',index=False)
 print('done')
 return test_df
start = time.time()
pairs_trans = create_record_for_features(sales, ['shop_id', 'item_id'], 'item_cnt_day', 'date_block_num', aggfunc = np.count_nonzero, fill = 0)
 print('It tooks: ', round(time.time() - start), 'seconds' )<define_variables>
md_ef = EfficientNet.from_pretrained('efficientnet-b5', num_classes=1)
 learn = Learner(data,
md_ef,
metrics = [qk],
callback_fns=[
BnFreeze,
partial(SaveModelCallback, monitor='quad_kappa', name='bestmodel')
 ],
model_dir="models", 
)
 if TTA:
learn = learn.to_fp32()
else:
learn = learn.to_fp16()
learn.data.add_test(ImageList.from_df(test_df,
'.. /input/aptos2019-blindness-detection',
folder='test_images',
suffix='.png'))
for month in range(1, 12):
pairs_temp = pairs_trans[['shop_id', 'item_id']][pairs_trans.loc[:,'0': str(month)].sum(axis = 1)== 0]
print('From month: 0 until ', month,', ', np.round(100 * len(pairs_temp)/ len(pairs_trans), 2), '% of the item/shop pairs have made no sales' )<define_variables>
!mkdir models
!cp '.. /input/kaggle-public/abcdef.pth' 'models'
learn.load('abcdef');
for month in range(21, 33):
pairs_temp = pairs_trans[['shop_id', 'item_id']][pairs_trans.loc[:,str(month): '33'].sum(axis = 1)== 0]
print('From month: ', month, ' until month: 33, ', np.round(100 * len(pairs_temp)/ len(pairs_trans), 2), '% of the item/shop pairs have made no sales' )<set_options>
learn.load('bestmodel' )
del pairs_trans
del pairs_temp
gc.collect()<prepare_x_and_y>
rounder = OptimizedRounder()
rounder.fit(valid_preds[0], valid_preds[1])
 rounder_coefficients = rounder.coefficients()
print(rounder_coefficients )
<define_variables><EOS> 
test_df_tta['diagnosis'].value_counts()