Kaggle competition: APTOS 2019 Blindness Detection
<SOS> metric: QuadraticWeightedKappa Kaggle data source: aptos-2019-blindness-detection<data_type_conversions>
%reload_ext autoreload
%autoreload 2
%matplotlib inline
%matplotlib inline
def downcast_dtypes(df):
start_size = df.memory_usage(deep = True ).sum() / 1024**2
print('Memory usage: {:.2f} MB'.format(start_size))
float_cols = [c for c in df if df[c].dtype == "float64"]
int_cols = [c for c in df if df[c].dtype in ["int64", "int32"]]
df[float_cols] = df[float_cols].astype(np.float32)
 df[int_cols] = df[int_cols].astype(np.int32)
 end_size = df.memory_usage(deep = True ).sum() / 1024**2
print('New Memory usage: {:.2f} MB'.format(end_size))
return df
def create_record_for_features(df, attrs, target, time_col, aggfunc = np.sum, fill = 0):
target_for_attrs = df.pivot_table(index = attrs,
values = target,
columns = time_col,
aggfunc = aggfunc,
fill_value = fill,
).reset_index()
target_for_attrs.columns = target_for_attrs.columns.map(str)
 target_for_attrs = target_for_attrs.reset_index(drop = True ).rename_axis(None, axis = 1)
 return target_for_attrs
def display_df_info(df, name):
print('-----------Shape of '+ name + '-------------')
 print(df.shape)
 print('-----------Missing values---------')
 print(df.isnull().sum())
 print('-----------Null values------------')
 print(df.isna().sum())
 print('-----------Data types-------------')
 print(df.dtypes)
 print('-----------Memory usage(MB)------')
 print(np.round(df.memory_usage(deep = True ).sum() / 1024**2, 2))<load_from_csv>
md_ef = EfficientNet.from_pretrained('efficientnet-b5', num_classes=1 )
sales = pd.read_csv(data_path + 'sales_train.csv')
 sales = downcast_dtypes(sales )<load_from_csv>
def get_df() :
base_image_dir = os.path.join('.. ', 'input/aptos2019-blindness-detection/')
 train_dir = os.path.join(base_image_dir,'train_images/')
 df = pd.read_csv(os.path.join(base_image_dir, 'train.csv'))
df['path'] = df['id_code'].map(lambda x: os.path.join(train_dir,'{}.png'.format(x)))
 df = df.drop(columns=['id_code'])
 df = df.sample(frac=1 ).reset_index(drop=True)
 test_df = pd.read_csv('.. /input/aptos2019-blindness-detection/sample_submission.csv')
 return df, test_df
df, test_df = get_df()
items = pd.read_csv(data_path + 'items.csv')
 items = downcast_dtypes(items )<load_from_csv>
bs = 128
sz = 256
tfms = get_transforms(do_flip=True, flip_vert=True )
item_categories = pd.read_csv(data_path + 'item_categories.csv')
 item_categories = downcast_dtypes(item_categories )<load_from_csv>
def qk(y_pred, y):
return torch.tensor(cohen_kappa_score(torch.round(y_pred), y, weights='quadratic'), device='cuda:0' )
shops = pd.read_csv(data_path + 'shops.csv')
 shops = downcast_dtypes(shops )<load_from_csv>
learn = Learner(data,
md_ef,
metrics = [qk],
model_dir="models" ).to_fp16()
learn.data.add_test(ImageList.from_df(test_df,
'.. /input/aptos2019-blindness-detection',
folder='test_images',
suffix='.png'))
test = pd.read_csv(data_path + 'test.csv')
 test = downcast_dtypes(test )<set_options>
learn.load('abcdef');
del sales_sampled
gc.collect()<feature_engineering>
class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
ll = metrics.cohen_kappa_score(y, X_p, weights='quadratic')
 return -ll
def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X=X, y=y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 print(-loss_partial(self.coef_['x']))
def predict(self, X, coef):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
return X_p
def coefficients(self):
return self.coef_['x']
<feature_engineering><EOS> 
def run_subm(learn=learn, coefficients=[0.5, 1.5, 2.5, 3.5]):
opt = OptimizedRounder()
preds,y = learn.get_preds(DatasetType.Test)
 tst_pred = opt.predict(preds, coefficients)
 test_df.diagnosis = tst_pred.astype(int)
 test_df.to_csv('submission.csv',index=False)
 print('done' )