Kaggle competition: APTOS 2019 Blindness Detection
<SOS> metric: QuadraticWeightedKappa Kaggle data source: aptos-2019-blindness-detection<feature_engineering>
%reload_ext autoreload
%autoreload 2
%matplotlib inline
%matplotlib inline
warnings.filterwarnings('ignore')
 dir_19_name = os.path.join('.. ', 'input/aptos2019-blindness-detection/')
 dir_15_name = os.path.join('.. ', 'input/diabetic-retinopathy-resized/')
 sys.path.append(os.path.abspath('.. /input/efficientnet-pytorch/efficientnet-pytorch/EfficientNet-PyTorch-master'))
not_know2['ConfirmedCases'] = not_know2['ConfirmedCases'].round()
not_know2['Fatalities'] = not_know2['Fatalities'].round()<sort_values>
md_ef = EfficientNet.from_name('efficientnet-b5',override_params={'num_classes':1})
 !mkdir models
!cp '.. /input/kaggle-public/abcdef.pth' 'models'
not_know2.loc[not_know2.Date=='2020-04-30',
['location','ConfirmedCases','Fatalities']].sort_values('ConfirmedCases', ascending=False ).head(20 )<save_to_csv>
def get_df(dir_15_name,dir_19_name):
valid_dir = os.path.join(dir_19_name,'train_images/')
 valid_df = pd.read_csv(os.path.join(dir_19_name,'train.csv'))
valid_df['path'] = valid_df['id_code'].map(lambda x: os.path.join(valid_dir,'{}.png'.format(x)))
 valid_df['is_valid'] = [True] * len(valid_df)
 valid_df = valid_df.sample(frac=1 ).reset_index(drop=True)
 test_df = pd.read_csv(os.path.join(dir_19_name,'sample_submission.csv'))
train_dir = os.path.join(dir_15_name,'resized_train_cropped/resized_train_cropped/')
 train_df = pd.read_csv(os.path.join(dir_15_name,'trainLabels_cropped.csv'))
train_df['path'] = train_df['image'].map(lambda x: os.path.join(train_dir,'{}.jpeg'.format(x)))
 train_df['diagnosis'] = train_df['level']
train_df['id_code'] = train_df['image']
train_df['is_valid'] = [False] * len(train_df)
 train_df1 = train_df[train_df['diagnosis'] == 0]
train_df2 = train_df[train_df['diagnosis'] != 0]
train_df1 = train_df1.sample(frac=1 ).reset_index(drop=True)
 train_df1 = train_df1[:5000]
train_df = pd.concat([train_df1,train_df2],axis=0,ignore_index=True)
 train_df = train_df.drop(columns = ['Unnamed: 0.1','Unnamed: 0','level','image'])
 train_df = train_df.sample(frac=1 ).reset_index(drop=True)
 return train_df, valid_df, test_df
train_df, valid_df, test_df = get_df(dir_15_name,dir_19_name)
 train_df1 = train_df[:4000]
train_df2 = train_df[4000:8000]
train_df3 = train_df[8000:12000]
train_df4 = train_df[12000:]
valid_df1 = valid_df[:1000]
valid_df2 = valid_df[1000:2000]
valid_df3 = valid_df[2000:3000]
valid_df4 = valid_df[3000:]
sub1 = pd.concat([know[['ForecastId', 'ConfirmedCases','Fatalities']],
not_know2[['ForecastId', 'ConfirmedCases','Fatalities']]])
 sub1=sub1.sort_values('ForecastId' ).reset_index(drop=True)
 sub1.to_csv('submission.csv', index=False )<load_from_csv>
res1 = pd.concat([train_df1,valid_df1],axis=0,ignore_index=True)
 res2 = pd.concat([train_df2,valid_df2],axis=0,ignore_index=True)
 res3 = pd.concat([train_df3,valid_df3],axis=0,ignore_index=True)
 res4 = pd.concat([train_df4,valid_df4],axis=0,ignore_index=True )
path_week3 = '/kaggle/input/covid19-global-forecasting-week-4'
df_train = pd.read_csv(f'{path_week3}/train.csv',parse_dates=['Date'], engine='python')
 df_test = pd.read_csv(f'{path_week3}/test.csv')
 df_train.head()
df_test.head()
df_train.rename(columns={'Country_Region':'Country'}, inplace=True)
 df_test.rename(columns={'Country_Region':'Country'}, inplace=True)
 df_train.rename(columns={'Province_State':'State'}, inplace=True)
 df_test.rename(columns={'Province_State':'State'}, inplace=True)
 df_train['Date'] = pd.to_datetime(df_train['Date'], infer_datetime_format=True)
 df_test['Date'] = pd.to_datetime(df_test['Date'], infer_datetime_format=True)
 df_train.info()
df_test.info()
y1_Train = df_train.iloc[:, -2]
y1_Train.head()
y2_Train = df_train.iloc[:, -1]
y2_Train.head()
EMPTY_VAL = "EMPTY_VAL"
def fillState(state, country):
if state == EMPTY_VAL: return country
return state
X_Train = df_train.copy()
X_Train['State'].fillna(EMPTY_VAL, inplace=True)
 X_Train['State'] = X_Train.loc[:, ['State', 'Country']].apply(lambda x : fillState(x['State'], x['Country']), axis=1)
 X_Train.loc[:, 'Date'] = X_Train.Date.dt.strftime("%m%d")
 X_Train["Date"] = X_Train["Date"].astype(int)
 X_Train.head()
X_Test = df_test.copy()
X_Test['State'].fillna(EMPTY_VAL, inplace=True)
 X_Test['State'] = X_Test.loc[:, ['State', 'Country']].apply(lambda x : fillState(x['State'], x['Country']), axis=1)
 X_Test.loc[:, 'Date'] = X_Test.Date.dt.strftime("%m%d")
 X_Test["Date"] = X_Test["Date"].astype(int)
 X_Test.head()
le = preprocessing.LabelEncoder()
X_Train.Country = le.fit_transform(X_Train.Country)
 X_Train['State'] = le.fit_transform(X_Train['State'])
 X_Train.head()
X_Test.Country = le.fit_transform(X_Test.Country)
 X_Test['State'] = le.fit_transform(X_Test['State'])
 X_Test.head()
df_train.head()
df_train.loc[df_train.Country == 'Afghanistan', :]
df_test.tail()
filterwarnings('ignore')
 le = preprocessing.LabelEncoder()
countries = X_Train.Country.unique()
df_out = pd.DataFrame({'ForecastId': [], 'ConfirmedCases': [], 'Fatalities': []})
 for country in countries:
states = X_Train.loc[X_Train.Country == country, :].State.unique()
for state in states:
X_Train_CS = X_Train.loc[(X_Train.Country == country)&(X_Train.State == state), ['State', 'Country', 'Date', 'ConfirmedCases', 'Fatalities']]
y1_Train_CS = X_Train_CS.loc[:, 'ConfirmedCases']
y2_Train_CS = X_Train_CS.loc[:, 'Fatalities']
X_Train_CS = X_Train_CS.loc[:, ['State', 'Country', 'Date']]
X_Train_CS.Country = le.fit_transform(X_Train_CS.Country)
 X_Train_CS['State'] = le.fit_transform(X_Train_CS['State'])
 X_Test_CS = X_Test.loc[(X_Test.Country == country)&(X_Test.State == state), ['State', 'Country', 'Date', 'ForecastId']]
X_Test_CS_Id = X_Test_CS.loc[:, 'ForecastId']
X_Test_CS = X_Test_CS.loc[:, ['State', 'Country', 'Date']]
X_Test_CS.Country = le.fit_transform(X_Test_CS.Country)
 X_Test_CS['State'] = le.fit_transform(X_Test_CS['State'])
 model1 = XGBRegressor(n_estimators=1000)
 model1.fit(X_Train_CS, y1_Train_CS)
 y1_pred = model1.predict(X_Test_CS)
 model2 = XGBRegressor(n_estimators=1000)
 model2.fit(X_Train_CS, y2_Train_CS)
 y2_pred = model2.predict(X_Test_CS)
 df = pd.DataFrame({'ForecastId': X_Test_CS_Id, 'ConfirmedCases': y1_pred, 'Fatalities': y2_pred})
 df_out = pd.concat([df_out, df], axis=0)
 df_out.ForecastId = df_out.ForecastId.astype('int')
 df_out.tail()
df_out.to_csv('submission.csv', index=False )<set_options>
batch_size = 64
image_size = 224
transforms = get_transforms(do_flip=True,flip_vert=True,max_rotate=360, max_zoom=1.3, max_warp=0.3, max_lighting=0.4,p_affine = 0.7 )
warnings.filterwarnings("ignore" )<load_from_csv>
preds = ['','','','','','','','','','']
i = 0
for sz in [224]:
test =(ImageList.from_df(test_df,
'.. /input/aptos2019-blindness-detection',
folder='test_images',
suffix='.png'))
data =(ImageList.from_df(df=valid_df,path='./',cols='path')
.split_by_rand_pct(0.2)
.label_from_df(cols='diagnosis',label_cls=FloatList)
.add_test(test)
.transform(transforms,size=sz,resize_method=ResizeMethod.SQUISH,padding_mode='zeros')
.databunch(bs=batch_size,num_workers=4)
.normalize(imagenet_stats)
)
 learn = Learner(data,
md_ef,
metrics = [qk],
model_dir="models" ).to_fp16()
learn.load('abcdef');
preds[i],y = learn.get_preds(DatasetType.Test)
 i = i + 1
preds_A =(preds[0] + preds[1] + preds[2] + preds[3] + preds[4] + preds[5] + preds[6] + preds[7] + preds[8] + preds[9])/10
df_train=pd.read_csv(".. /input/covid19-global-forecasting-week-4/train.csv")
 df_test=pd.read_csv(".. /input/covid19-global-forecasting-week-4/test.csv")
 df_sub=pd.read_csv(".. /input/covid19-global-forecasting-week-4/submission.csv")
 print(df_train.shape)
 print(df_test.shape)
 print(df_sub.shape )<count_unique_values>
!cp '.. /input/aptosdataset/19model2.pth' 'models'
vision.data.open_image = _load_format
data =(ImageList.from_df(df=res1,path='./',cols='path')
.split_from_df() 
.label_from_df(cols='diagnosis',label_cls=FloatList)
.transform(transforms,size=image_size,resize_method=ResizeMethod.SQUISH,padding_mode='zeros')
.databunch(bs=batch_size,num_workers=4)
.normalize(imagenet_stats)
)
 learn = Learner(data,
md_ef,
metrics = [qk],
callback_fns=[BnFreeze,
partial(SaveModelCallback, monitor='valid_loss', name='best_loss')],
model_dir="models" ).to_fp16()
learn.data.add_test(ImageList.from_df(test_df,
'.. /input/aptos2019-blindness-detection',
folder='test_images',
suffix='.png'))
learn.load('19model2')
 learn.fit_one_cycle(5,1e-3)
 learn.save('15model2')
 learn.unfreeze()
print(f"Unique Countries: {len(df_train.Country_Region.unique())}" )<count_unique_values>
data =(ImageList.from_df(df=res4,path='./',cols='path')
.split_from_df() 
.label_from_df(cols='diagnosis',label_cls=FloatList)
.transform(transforms,size=456,resize_method=ResizeMethod.SQUISH,padding_mode='zeros')
.databunch(bs=16,num_workers=4)
.normalize(imagenet_stats)
)
 learn = Learner(data,
md_ef,
metrics = [qk],
callback_fns=[BnFreeze,
partial(SaveModelCallback, monitor='valid_loss', name='15_final')],
model_dir="models" ).to_fp16()
learn.data.add_test(ImageList.from_df(test_df,
'.. /input/aptos2019-blindness-detection',
folder='test_images',
suffix='.png'))
learn.load('15_kappa_3')
 learn.fit_one_cycle(5,5e-5 )
print(f"Unique Regions: {df_train.shape[0]/len(df_train.Date.unique())}" )<count_values>
preds = ['','','','','','','','','','']
i = 0
opt = OptimizedRounder()
for sz in [264,232,240,248,256,264,232,240,248,256]:
test =(ImageList.from_df(test_df,
'.. /input/aptos2019-blindness-detection',
folder='test_images',
suffix='.png'))
data =(ImageList.from_df(df=valid_df,path='./',cols='path')
.split_by_rand_pct(0.2)
.label_from_df(cols='diagnosis',label_cls=FloatList)
.add_test(test)
.transform(transforms,size=sz,resize_method=ResizeMethod.SQUISH,padding_mode='zeros')
.databunch(bs=batch_size,num_workers=4)
.normalize(imagenet_stats)
)
 learn = Learner(data,
md_ef,
metrics = [quadratic_kappa],
model_dir="models" ).to_fp16()
learn.load('19_kappa');
preds[i],y = learn.get_preds(DatasetType.Test)
 i = i + 1
preds_B =(preds[0] + preds[1] + preds[2] + preds[3] + preds[4] + preds[5] + preds[6] + preds[7] + preds[8] + preds[9])/10
<count_missing_values><EOS> 
opt = OptimizedRounder()
tst_pred = opt.predict(preds_A * 0.5 + preds_B * 0.5,coef=[0.5, 1.5, 2.5, 3.5])
 test_df.diagnosis = tst_pred.astype(int)
 test_df.to_csv('submission.csv',index=False)
 print('done' )