Kaggle competition: APTOS 2019 Blindness Detection
<SOS> metric: QuadraticWeightedKappa Kaggle data source: aptos-2019-blindness-detection<set_options>
%reload_ext autoreload
%autoreload 2
%matplotlib inline
plt.style.use('dark_background')
 warnings.filterwarnings('ignore')
<load_from_csv>
class RAdam(Optimizer):
def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8, weight_decay=0):
defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay)
 self.buffer = [[None, None, None] for ind in range(10)]
super(RAdam, self ).__init__(params, defaults)
 def __setstate__(self, state):
super(RAdam, self ).__setstate__(state)
 def step(self, closure=None):
loss = None
if closure is not None:
loss = closure()
for group in self.param_groups:
for p in group['params']:
if p.grad is None:
continue
grad = p.grad.data.float()
if grad.is_sparse:
raise RuntimeError('RAdam does not support sparse gradients')
 p_data_fp32 = p.data.float()
state = self.state[p]
if len(state)== 0:
state['step'] = 0
state['exp_avg'] = torch.zeros_like(p_data_fp32)
 state['exp_avg_sq'] = torch.zeros_like(p_data_fp32)
 else:
state['exp_avg'] = state['exp_avg'].type_as(p_data_fp32)
 state['exp_avg_sq'] = state['exp_avg_sq'].type_as(p_data_fp32)
 exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq']
beta1, beta2 = group['betas']
exp_avg_sq.mul_(beta2 ).addcmul_(1 - beta2, grad, grad)
 exp_avg.mul_(beta1 ).add_(1 - beta1, grad)
 state['step'] += 1
buffered = self.buffer[int(state['step'] % 10)]
if state['step'] == buffered[0]:
N_sma, step_size = buffered[1], buffered[2]
else:
buffered[0] = state['step']
beta2_t = beta2 ** state['step']
N_sma_max = 2 /(1 - beta2)- 1
N_sma = N_sma_max - 2 * state['step'] * beta2_t /(1 - beta2_t)
 buffered[1] = N_sma
if N_sma >= 5:
step_size = math.sqrt(( 1 - beta2_t)*(N_sma - 4)/(N_sma_max - 4)*(N_sma - 2)/ N_sma * N_sma_max /(N_sma_max - 2)) /(1 - beta1 ** state['step'])
 else:
step_size = 1.0 /(1 - beta1 ** state['step'])
 buffered[2] = step_size
if group['weight_decay'] != 0:
p_data_fp32.add_(-group['weight_decay'] * group['lr'], p_data_fp32)
 if N_sma >= 5:
denom = exp_avg_sq.sqrt().add_(group['eps'])
 p_data_fp32.addcdiv_(-step_size * group['lr'], exp_avg, denom)
 else:
p_data_fp32.add_(-step_size * group['lr'], exp_avg)
 p.data.copy_(p_data_fp32)
 return loss
class PlainRAdam(Optimizer):
def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8, weight_decay=0):
defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay)
 super(PlainRAdam, self ).__init__(params, defaults)
 def __setstate__(self, state):
super(PlainRAdam, self ).__setstate__(state)
 def step(self, closure=None):
loss = None
if closure is not None:
loss = closure()
for group in self.param_groups:
for p in group['params']:
if p.grad is None:
continue
grad = p.grad.data.float()
if grad.is_sparse:
raise RuntimeError('RAdam does not support sparse gradients')
 p_data_fp32 = p.data.float()
state = self.state[p]
if len(state)== 0:
state['step'] = 0
state['exp_avg'] = torch.zeros_like(p_data_fp32)
 state['exp_avg_sq'] = torch.zeros_like(p_data_fp32)
 else:
state['exp_avg'] = state['exp_avg'].type_as(p_data_fp32)
 state['exp_avg_sq'] = state['exp_avg_sq'].type_as(p_data_fp32)
 exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq']
beta1, beta2 = group['betas']
exp_avg_sq.mul_(beta2 ).addcmul_(1 - beta2, grad, grad)
 exp_avg.mul_(beta1 ).add_(1 - beta1, grad)
 state['step'] += 1
beta2_t = beta2 ** state['step']
N_sma_max = 2 /(1 - beta2)- 1
N_sma = N_sma_max - 2 * state['step'] * beta2_t /(1 - beta2_t)
 if group['weight_decay'] != 0:
p_data_fp32.add_(-group['weight_decay'] * group['lr'], p_data_fp32)
 if N_sma >= 5:
step_size = group['lr'] * math.sqrt(( 1 - beta2_t)*(N_sma - 4)/(N_sma_max - 4)*(N_sma - 2)/ N_sma * N_sma_max /(N_sma_max - 2)) /(1 - beta1 ** state['step'])
 denom = exp_avg_sq.sqrt().add_(group['eps'])
 p_data_fp32.addcdiv_(-step_size, exp_avg, denom)
 else:
step_size = group['lr'] /(1 - beta1 ** state['step'])
 p_data_fp32.add_(-step_size, exp_avg)
 p.data.copy_(p_data_fp32)
 return loss
class AdamW(Optimizer):
def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8, weight_decay=0, warmup = 0):
defaults = dict(lr=lr, betas=betas, eps=eps,
weight_decay=weight_decay, warmup = warmup)
 super(AdamW, self ).__init__(params, defaults)
 def __setstate__(self, state):
super(AdamW, self ).__setstate__(state)
 def step(self, closure=None):
loss = None
if closure is not None:
loss = closure()
for group in self.param_groups:
for p in group['params']:
if p.grad is None:
continue
grad = p.grad.data.float()
if grad.is_sparse:
raise RuntimeError('Adam does not support sparse gradients, please consider SparseAdam instead')
 p_data_fp32 = p.data.float()
state = self.state[p]
if len(state)== 0:
state['step'] = 0
state['exp_avg'] = torch.zeros_like(p_data_fp32)
 state['exp_avg_sq'] = torch.zeros_like(p_data_fp32)
 else:
state['exp_avg'] = state['exp_avg'].type_as(p_data_fp32)
 state['exp_avg_sq'] = state['exp_avg_sq'].type_as(p_data_fp32)
 exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq']
beta1, beta2 = group['betas']
state['step'] += 1
exp_avg_sq.mul_(beta2 ).addcmul_(1 - beta2, grad, grad)
 exp_avg.mul_(beta1 ).add_(1 - beta1, grad)
 denom = exp_avg_sq.sqrt().add_(group['eps'])
 bias_correction1 = 1 - beta1 ** state['step']
bias_correction2 = 1 - beta2 ** state['step']
if group['warmup'] > state['step']:
scheduled_lr = 1e-8 + state['step'] * group['lr'] / group['warmup']
else:
scheduled_lr = group['lr']
step_size = group['lr'] * math.sqrt(bias_correction2)/ bias_correction1
if group['weight_decay'] != 0:
p_data_fp32.add_(-group['weight_decay'] * scheduled_lr, p_data_fp32)
 p_data_fp32.addcdiv_(-step_size, exp_avg, denom)
 p.data.copy_(p_data_fp32)
 return loss
train_data=pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/train.csv')
 test_data=pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/test.csv' )<rename_columns>
print('Make sure cudnn is enabled:', torch.backends.cudnn.enabled )
train_data.rename(columns={'Country_Region':'Country'}, inplace=True)
 test_data.rename(columns={'Country_Region':'Country'}, inplace=True)
 train_data.rename(columns={'Province_State':'Province'}, inplace=True)
 test_data.rename(columns={'Province_State':'Province'}, inplace=True )<data_type_conversions>
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
SEED = 999
seed_everything(SEED )
train_data['Date'] = pd.to_datetime(train_data['Date'])
 test_data['Date'] = pd.to_datetime(test_data['Date'] )<data_type_conversions>
base_image_dir = os.path.join('.. ', 'input/aptos2019-blindness-detection/')
 train_dir = os.path.join(base_image_dir,'train_images/')
 df = pd.read_csv(os.path.join(base_image_dir, 'train.csv'))
df['path'] = df['id_code'].map(lambda x: os.path.join(train_dir,'{}.png'.format(x)))
 df = df.drop(columns=['id_code'])
 df = df.sample(frac=1 ).reset_index(drop=True)
 df.head(5 )
train_data['Date']=train_data['Date'].dt.strftime('%m%d')
 test_data['Date']=test_data['Date'].dt.strftime('%m%d')
 train_data['Date']=train_data['Date'].astype(int)
 test_data['Date']=test_data['Date'].astype(int )<feature_engineering>
df['diagnosis'].value_counts()
train_data["Province"]=train_data.apply(lambda row: str(row['Country'])if pd.isnull(row['Province'])else row['Province'] ,axis=1)
 test_data["Province"]=test_data.apply(lambda row: str(row['Country'])if pd.isnull(row['Province'])else row['Province'] ,axis=1 )<categorify>
im = Image.open(df['path'][1])
 width, height = im.size
print(width,height)
 im.show()
plt.imshow(np.asarray(im))
encoder=LabelEncoder()
train_data['Province']=encoder.fit_transform(train_data['Province'])
 test_data['Province']=encoder.fit_transform(test_data['Province'])
 train_data['Country']=encoder.fit_transform(train_data['Country'])
 test_data['Country']=encoder.fit_transform(test_data['Country'] )<drop_column>
bs = 64
sz = 224
train_data.drop(['Id'],axis=1,inplace=True )<define_variables>
class MyImageItemList(ImageList):
def open(self, fn:PathOrStr)->Image:
img = load_ben_color(fn.replace('/./','' ).replace('//','/'))
return vision.Image(pil2tensor(img, np.float32 ).div_(255))
tfms = get_transforms(do_flip=True,flip_vert=True,max_rotate=360,max_warp=0,max_zoom=1.1,max_lighting=0.1,p_lighting=0.5)
 data =(MyImageItemList.from_df(df=df,path='./',cols='path')
.split_by_rand_pct(0.2)
.label_from_df(cols='diagnosis',label_cls=FloatList)
.transform(tfms,size=sz,resize_method=ResizeMethod.SQUISH,padding_mode='zeros')
.databunch(bs=bs,num_workers=4)
.normalize(imagenet_stats))
features=['Province','Country','Date']
target_1=['ConfirmedCases']
target_2=['Fatalities']<normalization>
data.show_batch(rows=3, figsize=(7,7))
scaler = MinMaxScaler()<data_type_conversions>
def quadratic_kappa(y_hat, y):
return torch.tensor(cohen_kappa_score(torch.round(y_hat), y, weights='quadratic'),device='cuda:0' )
train_data[target_1]=train_data[target_1].astype(int)
 train_data[target_2]=train_data[target_2].astype(int )<choose_model_class>
learn = cnn_learner(data, base_arch=models.resnet50, metrics=[quadratic_kappa])
 learn.summary()
xgb_clf_1=XGBRegressor(n_estimators=1300)
 xgb_clf_2=XGBRegressor(n_estimators=1300 )<train_model>
learn.unfreeze()
learn.lr_find()
learn.recorder.plot(suggestion=True )
xgb_df=pd.DataFrame({'ForecastId':[],'ConfirmedCases':[],'Fatalities':[]})
 for country in train_data['Country'].unique() :
Province=train_data.loc[train_data.Country == country, :].Province.unique().tolist()
for province in Province:
new_train=train_data.loc[train_data['Province']==province]
new_train[features] = scaler.fit_transform(new_train[features].values)
 xgb_clf_1.fit(new_train[features],new_train[target_1])
 xgb_clf_2.fit(new_train[features],new_train[target_2])
 new_test=test_data.loc[test_data['Province']==province]
new_test_FI=new_test.ForecastId
new_test[features]=scaler.transform(new_test[features])
 new_pred_1=xgb_clf_1.predict(new_test[features])
 new_pred_2=xgb_clf_2.predict(new_test[features])
 new_df=pd.DataFrame({'ForecastId':new_test_FI,'ConfirmedCases':new_pred_1,'Fatalities':new_pred_2})
 xgb_df=pd.concat([xgb_df,new_df],axis=0 )<feature_engineering>
learn.fit_one_cycle(10, max_lr=slice(1e-6,1e-3))
xgb_df=xgb_df.drop_duplicates()
xgb_df['ForecastId']=xgb_df['ForecastId'].astype(int)
 xgb_df['ConfirmedCases']=np.round(xgb_df['ConfirmedCases'])
 xgb_df['Fatalities']=np.round(xgb_df['Fatalities'] )<save_to_csv>
learn.export()
learn.save('stage-2' )
xgb_df.to_csv('submission.csv',index=False )<set_options>
preds, y, losses = learn.get_preds(with_loss=True )
plt.style.use('ggplot')
 font = {'family' : 'meiryo'}
plt.rc('font', **font )<set_options>
interp = ClassificationInterpretation.from_learner(learn)
 losses, idxs = interp.top_losses()
os.environ['PYTHONHASHSEED'] = '0'
np.random.seed(42)
 rn.seed(12345)
 session_conf = tf.compat.v1.ConfigProto(intra_op_parallelism_threads=1, inter_op_parallelism_threads=1)
 tf.random.set_seed(1234)
 sess = tf.compat.v1.Session(graph=tf.compat.v1.get_default_graph() , config=session_conf)
 tf.compat.v1.keras.backend.set_session(sess )<load_from_csv>
valid_preds = learn.get_preds(ds_type=DatasetType.Valid )
train_df = pd.read_csv(".. /input/covid19-global-forecasting-week-4/train.csv")
 train_df = train_df.fillna("No State")
 train_df<prepare_x_and_y>
import numpy as np
import pandas as pd
import os
import scipy as sp
from functools import partial
from sklearn import metrics
from collections import Counter
import json
test_rate = 0.1
time_series_len = 18
train_date_count = len(set(train_df["Date"]))
X, Y = [],[]
scaler = StandardScaler()
train_df["ConfirmedCases_std"] = scaler.fit_transform(train_df["ConfirmedCases"].values.reshape(len(train_df["ConfirmedCases"].values),1))
for state,country in train_df.groupby(["Province_State","Country_Region"] ).sum().index:
df = train_df[(train_df["Country_Region"] == country)&(train_df["Province_State"] == state)]
if df["ConfirmedCases"].sum() != 0:
for i in range(len(df)- time_series_len):
X.append(df[['ConfirmedCases_std']].iloc[i:(i+time_series_len)].values)
 Y.append(df[['ConfirmedCases_std']].iloc[i+time_series_len].values)
 X=np.array(X)
 Y=np.array(Y)
 X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=test_rate, shuffle = True ,random_state = 0 )<compute_test_metric>
class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
ll = metrics.cohen_kappa_score(y, X_p, weights='quadratic')
 return -ll
def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X=X, y=y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 print(-loss_partial(self.coef_['x']))
def predict(self, X, coef):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
return X_p
def coefficients(self):
return self.coef_['x']
def huber_loss(y_true, y_pred, clip_delta=1.0):
error = y_true - y_pred
cond = tf.keras.backend.abs(error)< clip_delta
squared_loss = 0.5 * tf.keras.backend.square(error)
 linear_loss = clip_delta *(tf.keras.backend.abs(error)- 0.5 * clip_delta)
 return tf.where(cond, squared_loss, linear_loss)
 def huber_loss_mean(y_true, y_pred, clip_delta=1.0):
return tf.keras.backend.mean(huber_loss(y_true, y_pred, clip_delta))<choose_model_class>
optR = OptimizedRounder()
optR.fit(valid_preds[0],valid_preds[1] )
epochs_num = 20
n_in = 1
model = Sequential()
model.add(GRU(100,
batch_input_shape=(None, time_series_len, n_in),
kernel_initializer=random_uniform(seed=1),
return_sequences=False 
))
model.add(Dense(50))
model.add(Dropout(0.15))
model.add(Dense(n_in, kernel_initializer=random_uniform(seed=1)))
 model.add(Activation("linear"))
opt = Adagrad(lr=0.01, epsilon=1e-08, decay=1e-4)
 model.compile(loss = huber_loss_mean, optimizer=opt )<train_model>
sample_df = pd.read_csv('.. /input/aptos2019-blindness-detection/sample_submission.csv')
 sample_df.head()
callbacks = [ReduceLROnPlateau(monitor='loss', patience=4, verbose=1, factor=0.6),
EarlyStopping(monitor='loss', patience=10)]
hist = model.fit(X_train, Y_train, batch_size=20, epochs=epochs_num,
callbacks=callbacks,shuffle=False )<predict_on_test>
learn.data.add_test(MyImageItemList.from_df(sample_df,'.. /input/aptos2019-blindness-detection',folder='test_images',suffix='.png'))
predicted_std = model.predict(X_test)
 result_std= pd.DataFrame(predicted_std)
 result_std.columns = ['predict']
result_std['actual'] = Y_test
result_std.plot(figsize=(25,6))
plt.show()<normalization>
preds, y = learn.get_preds(ds_type=DatasetType.Test )
predicted = scaler.inverse_transform(predicted_std)
 Y_test2 = scaler.inverse_transform(Y_test )<compute_test_metric>
test_predictions = optR.predict(preds, coefficients )
np.sqrt(mean_squared_log_error(predicted, Y_test2))<create_dataframe>
sample_df.diagnosis = test_predictions.astype(int)
 sample_df.head()
result= pd.DataFrame(predicted)
 result.columns = ['predict']
result['actual'] = Y_test2
result.plot(figsize=(25,6))
plt.show()<load_from_csv>
sample_df.to_csv('submission.csv',index=False )