Kaggle competition: APTOS 2019 Blindness Detection
<SOS> metric: QuadraticWeightedKappa Kaggle data source: aptos-2019-blindness-detection<feature_engineering>
t_start = time.time()
df_train["UniqueRegion"]=df_train.Country_Region
df_train.UniqueRegion[df_train.Province_State.isna() ==False]=df_train.Province_State+" , "+df_train.Country_Region
df_train[df_train.Province_State.isna() ==False]<drop_column>
IMG_WIDTH = 512
IMG_HEIGHT = 512
CHANNEL = 3
BATCH_SIZE = 4
OLD_DATA_EPOCHS = 6
NEW_DATA_EPOCHS = 6
WARMUP_EPOCHS = 3
NUM_CLASSES = 5
SEED = 2
n_folds = 1
ES_PATIENCE = 5
RLROP_PATIENCE = 2
DECAY_DROP = 0.2
df_train.drop(labels=["Id","Province_State","Country_Region"], axis=1, inplace=True )<feature_engineering>
def append_file_ext_png(file_name):
return file_name + ".png"
def append_file_ext_jpeg(file_name):
return file_name + ".jpeg"
test_dates=list(df_test.Date.unique())
 print(f"Period :{len(df_test.Date.unique())} days")
 print(f"From : {df_test.Date.min() } To : {df_test.Date.max() }" )<define_variables>
BASE_DIR = '/kaggle/input/aptos2019-blindness-detection/'
TRAIN_DIR = '/kaggle/input/diabetic-retinopathy-resized/resized_train/resized_train'
TEST_DIR = '/kaggle/input/aptos2019-blindness-detection/test_images'
print(f"Total Regions : {df_test.shape[0]/43}" )<feature_engineering>
TEST_DF = pd.read_csv(BASE_DIR + "test.csv",dtype='object')
 TRAIN_DF = pd.read_csv("/kaggle/input/diabetic-retinopathy-resized/trainLabels.csv",dtype='object')
 TRAIN_DF.columns = ['id_code', 'diagnosis']
X_COL='id_code'
Y_COL='diagnosis'
TRAIN_DF[X_COL] = TRAIN_DF[X_COL].apply(append_file_ext_jpeg)
 TEST_DF[X_COL] = TEST_DF[X_COL].apply(append_file_ext_png )
df_test["UniqueRegion"]=df_test.Country_Region
df_test.UniqueRegion[df_test.Province_State.isna() ==False]=df_test.Province_State+" , "+df_test.Country_Region
df_test.drop(labels=["Province_State","Country_Region"], axis=1, inplace=True )<count_unique_values>
df0 = TRAIN_DF.loc[TRAIN_DF['diagnosis'] == '0']
df1 = TRAIN_DF.loc[TRAIN_DF['diagnosis'] == '1']
df2 = TRAIN_DF.loc[TRAIN_DF['diagnosis'] == '2']
df3 = TRAIN_DF.loc[TRAIN_DF['diagnosis'] == '3']
df4 = TRAIN_DF.loc[TRAIN_DF['diagnosis'] == '4']
df0 = df0.head(2000)
 df1 = df1.head(2000)
 df2 = df2.head(2000)
 TRAIN_DF = df0.append([df1, df2, df3, df4],ignore_index = True)
 TRAIN_DF = shuffle(TRAIN_DF )
len(df_test.UniqueRegion.unique() )<define_variables>
class RAdam(keras.optimizers.Optimizer):

 def __init__(self, lr=0.001, beta_1=0.9, beta_2=0.999,
epsilon=None, decay=0., weight_decay=0., amsgrad=False,
total_steps=0, warmup_proportion=0.1, min_lr=0., **kwargs):
super(RAdam, self ).__init__(**kwargs)
 with K.name_scope(self.__class__.__name__):
self.iterations = K.variable(0, dtype='int64', name='iterations')
 self.lr = K.variable(lr, name='lr')
 self.beta_1 = K.variable(beta_1, name='beta_1')
 self.beta_2 = K.variable(beta_2, name='beta_2')
 self.decay = K.variable(decay, name='decay')
 self.weight_decay = K.variable(weight_decay, name='weight_decay')
 self.total_steps = K.variable(total_steps, name='total_steps')
 self.warmup_proportion = K.variable(warmup_proportion, name='warmup_proportion')
 self.min_lr = K.variable(lr, name='min_lr')
 if epsilon is None:
epsilon = K.epsilon()
self.epsilon = epsilon
self.initial_decay = decay
self.initial_weight_decay = weight_decay
self.initial_total_steps = total_steps
self.amsgrad = amsgrad
def get_updates(self, loss, params):
grads = self.get_gradients(loss, params)
 self.updates = [K.update_add(self.iterations, 1)]
lr = self.lr
if self.initial_decay > 0:
lr = lr *(1./(1.+ self.decay * K.cast(self.iterations, K.dtype(self.decay))))
t = K.cast(self.iterations, K.floatx())+ 1
if self.initial_total_steps > 0:
warmup_steps = self.total_steps * self.warmup_proportion
decay_steps = self.total_steps - warmup_steps
lr = K.switch(
 t <= warmup_steps,
lr *(t / warmup_steps),
lr *(1.0 - K.minimum(t, decay_steps)/ decay_steps), 
)
 ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p), name='m_' + str(i)) for(i, p)in enumerate(params)]
vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p), name='v_' + str(i)) for(i, p)in enumerate(params)]
if self.amsgrad:
vhats = [K.zeros(K.int_shape(p), dtype=K.dtype(p), name='vhat_' + str(i)) for(i, p)in enumerate(params)]
else:
vhats = [K.zeros(1, name='vhat_' + str(i)) for i in range(len(params)) ]
self.weights = [self.iterations] + ms + vs + vhats
beta_1_t = K.pow(self.beta_1, t)
 beta_2_t = K.pow(self.beta_2, t)
 sma_inf = 2.0 /(1.0 - self.beta_2)- 1.0
sma_t = sma_inf - 2.0 * t * beta_2_t /(1.0 - beta_2_t)
 for p, g, m, v, vhat in zip(params, grads, ms, vs, vhats):
m_t =(self.beta_1 * m)+(1.- self.beta_1)* g
v_t =(self.beta_2 * v)+(1.- self.beta_2)* K.square(g)
 m_corr_t = m_t /(1.0 - beta_1_t)
 if self.amsgrad:
vhat_t = K.maximum(vhat, v_t)
 v_corr_t = K.sqrt(vhat_t /(1.0 - beta_2_t)+ self.epsilon)
 self.updates.append(K.update(vhat, vhat_t))
else:
v_corr_t = K.sqrt(v_t /(1.0 - beta_2_t)+ self.epsilon)
 r_t = K.sqrt(( sma_t - 4.0)/(sma_inf - 4.0)* 
(sma_t - 2.0)/(sma_inf - 2.0)*
sma_inf / sma_t)
 p_t = K.switch(sma_t > 5, r_t * m_corr_t / v_corr_t, m_corr_t)
 if self.initial_weight_decay > 0:
p_t += self.weight_decay * p
p_t = p - lr * p_t
self.updates.append(K.update(m, m_t))
self.updates.append(K.update(v, v_t))
new_p = p_t
if getattr(p, 'constraint', None)is not None:
new_p = p.constraint(new_p)
 self.updates.append(K.update(p, new_p))
return self.updates
def get_config(self):
config = {
'lr': float(K.get_value(self.lr)) ,
'beta_1': float(K.get_value(self.beta_1)) ,
'beta_2': float(K.get_value(self.beta_2)) ,
'decay': float(K.get_value(self.decay)) ,
'weight_decay': float(K.get_value(self.weight_decay)) ,
'epsilon': self.epsilon,
'amsgrad': self.amsgrad,
'total_steps': float(K.get_value(self.total_steps)) ,
'warmup_proportion': float(K.get_value(self.warmup_proportion)) ,
'min_lr': float(K.get_value(self.min_lr)) ,
}
base_config = super(RAdam, self ).get_config()
return dict(list(base_config.items())+ list(config.items()))
only_train_dates=set(train_dates)-set(test_dates)
 print("Only train dates : ",len(only_train_dates))
intersection_dates=set(test_dates)&set(train_dates)
 print("Intersection dates : ",len(intersection_dates))
only_test_dates=set(test_dates)-set(train_dates)
 print("Only Test dates : ",len(only_test_dates))<feature_engineering>
def create_model(input_shape, n_out):
input_tensor = Input(shape=input_shape)
 base_model = EfficientNetB5(weights=None,
include_top=False,
input_tensor=input_tensor)
 base_model.load_weights('/kaggle/input/efficientnet-keras-weights-b0b5/efficientnet-b5_imagenet_1000_notop.h5')
 x = GlobalAveragePooling2D()(base_model.output)
 x = Dropout(0.5 )(x)
 x = Dense(2048, activation='relu' )(x)
 x = Dropout(0.5 )(x)
 x = Dense(1024, activation='relu' )(x)
 final_output = Dense(n_out, activation='softmax', name='final_output' )(x)
 model = Model(input_tensor, final_output)
 return model
df_test_temp=pd.DataFrame()
df_test_temp["Date"]=df_test.Date
df_test_temp["ConfirmedCases"]=0.0
df_test_temp["Fatalities"]=0.0
df_test_temp["UniqueRegion"]=df_test.UniqueRegion
df_test_temp["Delta"]=1.0<feature_engineering>
es = EarlyStopping(monitor='val_loss',
mode='min',
patience=ES_PATIENCE,
restore_best_weights=True,
verbose=1)
 rlrop = ReduceLROnPlateau(monitor='val_loss',
mode='min',
patience=RLROP_PATIENCE,
factor=DECAY_DROP,
verbose=1)
 model_checkpoint = ModelCheckpoint('EfficientNetB5.h5',
verbose=1,
save_best_only=True)
 callback_list = [es, rlrop, model_checkpoint]
%%time
final_df=pd.DataFrame(columns=["Date","ConfirmedCases","Fatalities","UniqueRegion"])
 for region in df_train.UniqueRegion.unique() :
df_temp=df_train[df_train.UniqueRegion==region].reset_index()
df_temp["Delta"]=1.0
size_train=df_temp.shape[0]
for i in range(1,df_temp.shape[0]):
if(df_temp.ConfirmedCases[i-1]>0):
df_temp.Delta[i]=df_temp.ConfirmedCases[i]/df_temp.ConfirmedCases[i-1]
n=7
delta_list=df_temp.tail(n ).Delta
delta_avg=df_temp.tail(n ).Delta.mean()
death_rate=df_temp.tail(1 ).Fatalities.sum() /df_temp.tail(1 ).ConfirmedCases.sum()
df_test_app=df_test_temp[df_test_temp.UniqueRegion==region]
df_test_app=df_test_app[df_test_app.Date>df_temp.Date.max() ]
X=np.arange(1,n+1 ).reshape(-1,1)
 Y=delta_list
model=LinearRegression()
model.fit(X,Y)
 df_temp=pd.concat([df_temp,df_test_app])
 df_temp=df_temp.reset_index()
for i in range(size_train, df_temp.shape[0]):
n=n+1
damper=df_temp.Delta[i-5]
pred=max(1,model.predict(np.array([n] ).reshape(-1,1)) [0])
 df_temp.Delta[i]=(damper+pred+delta_avg)/3
for i in range(size_train, df_temp.shape[0]):
df_temp.ConfirmedCases[i]=round(df_temp.ConfirmedCases[i-1]*df_temp.Delta[i],0)
 df_temp.Fatalities[i]=round(death_rate*df_temp.ConfirmedCases[i],0)
 size_test=df_temp.shape[0]-df_test_temp[df_test_temp.UniqueRegion==region].shape[0]
df_temp=df_temp.iloc[size_test:,:]
df_temp=df_temp[["Date","ConfirmedCases","Fatalities","UniqueRegion","Delta"]]
final_df=pd.concat([final_df,df_temp], ignore_index=True)
 final_df.shape<save_to_csv>
datagen = ImageDataGenerator(
 rescale=1./255.,
validation_split=0.25)
 train_generator = datagen.flow_from_dataframe(
 dataframe=TRAIN_DF,
directory=TRAIN_DIR,
x_col=X_COL,
y_col=Y_COL,
subset="training",
batch_size=BATCH_SIZE,
seed=SEED,
zoom_range=0.2,
horizontal_flip=True,
vertical_flip=True,
class_mode="categorical",
preprocessing_function=preprocess_image,
target_size=(IMG_WIDTH,IMG_HEIGHT))
valid_generator=datagen.flow_from_dataframe(
 dataframe=TRAIN_DF,
directory=TRAIN_DIR,
x_col=X_COL,
y_col=Y_COL,
subset="validation",
batch_size=BATCH_SIZE,
seed=SEED,
zoom_range=0.2,
horizontal_flip=True,
vertical_flip=True,
class_mode="categorical",
preprocessing_function=preprocess_image,
target_size=(IMG_WIDTH,IMG_HEIGHT))
STEP_SIZE_TRAIN=train_generator.n//train_generator.batch_size
STEP_SIZE_VALID=valid_generator.n//valid_generator.batch_size
df_sub.Fatalities=final_df.Fatalities
df_sub.ConfirmedCases=final_df.ConfirmedCases
df_sub.to_csv("submission.csv", index=None )<load_from_csv>
model = create_model(input_shape=(IMG_WIDTH, IMG_HEIGHT, CHANNEL), n_out=NUM_CLASSES)
 for layer in model.layers:
layer.trainable = False
for i in range(-10, 0):
model.layers[i].trainable = True
metric_list = ["accuracy"]
optimizer = RAdam(lr=0.00005)
 model.compile(optimizer=optimizer, loss="categorical_crossentropy", metrics=metric_list )
train = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/train.csv')
 test = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/test.csv' )<data_type_conversions>
history_warmup = model.fit_generator(generator=train_generator,
steps_per_epoch=STEP_SIZE_TRAIN,
validation_data=valid_generator,
validation_steps=STEP_SIZE_VALID,
epochs=WARMUP_EPOCHS,
verbose=1 ).history
train.rename(columns={'Country_Region':'Country'}, inplace=True)
 test.rename(columns={'Country_Region':'Country'}, inplace=True)
 train.rename(columns={'Province_State':'State'}, inplace=True)
 test.rename(columns={'Province_State':'State'}, inplace=True)
 train['Date'] = pd.to_datetime(train['Date'], infer_datetime_format=True)
 test['Date'] = pd.to_datetime(test['Date'], infer_datetime_format=True)
 train.info()
test.info()<something_strange>
for layer in model.layers:
layer.trainable = True
optimizer = RAdam(lr=0.00005)
 model.compile(optimizer=optimizer, loss="binary_crossentropy", metrics=metric_list)
 gc.collect()
history_finetunning = model.fit_generator(generator=train_generator,
steps_per_epoch=STEP_SIZE_TRAIN,
validation_data=valid_generator,
validation_steps=STEP_SIZE_VALID,
epochs=OLD_DATA_EPOCHS,
callbacks=callback_list,
verbose=1 ).history
EMPTY_VAL = "EMPTY_VAL"
def fillState(state, country):
if state == EMPTY_VAL: return country
return state<data_type_conversions>
NEW_TRAIN_DIR = '/kaggle/input/aptos2019-blindness-detection/train_images'
NEW_TRAIN_DF = pd.read_csv(BASE_DIR + "train.csv",dtype='object')
 NEW_TRAIN_DF[X_COL] = NEW_TRAIN_DF[X_COL].apply(append_file_ext_png)
 NEW_TRAIN_DF.head()
X_xTrain = train.copy()
X_xTrain['State'].fillna(EMPTY_VAL, inplace=True)
 X_xTrain['State'] = X_xTrain.loc[:, ['State', 'Country']].apply(lambda x : fillState(x['State'], x['Country']), axis=1)
 X_xTrain.loc[:, 'Date'] = X_xTrain.Date.dt.strftime("%m%d")
 X_xTrain["Date"] = X_xTrain["Date"].astype(int)
 X_xTrain.tail()<data_type_conversions>
datagen = ImageDataGenerator(
 rescale=1./255.,
validation_split=0.25)
 train_generator = datagen.flow_from_dataframe(
 dataframe=NEW_TRAIN_DF,
directory=NEW_TRAIN_DIR,
x_col=X_COL,
y_col=Y_COL,
subset="training",
batch_size=BATCH_SIZE,
seed=SEED,
zoom_range=0.2,
horizontal_flip=True,
vertical_flip=True,
class_mode="categorical",
preprocessing_function=preprocess_image,
target_size=(IMG_WIDTH,IMG_HEIGHT))
valid_generator=datagen.flow_from_dataframe(
 dataframe=NEW_TRAIN_DF,
directory=NEW_TRAIN_DIR,
x_col=X_COL,
y_col=Y_COL,
subset="validation",
batch_size=BATCH_SIZE,
seed=SEED,
zoom_range=0.2,
horizontal_flip=True,
vertical_flip=True,
class_mode="categorical",
preprocessing_function=preprocess_image,
target_size=(IMG_WIDTH,IMG_HEIGHT))
STEP_SIZE_TRAIN=train_generator.n//train_generator.batch_size
STEP_SIZE_VALID=valid_generator.n//valid_generator.batch_size
X_xTest = test.copy()
X_xTest['State'].fillna(EMPTY_VAL, inplace=True)
 X_xTest['State'] = X_xTest.loc[:, ['State', 'Country']].apply(lambda x : fillState(x['State'], x['Country']), axis=1)
 X_xTest.loc[:, 'Date'] = X_xTest.Date.dt.strftime("%m%d")
 X_xTest["Date"] = X_xTest["Date"].astype(int)
 X_xTest.tail()<categorify>
gc.collect()
history_finetunning = model.fit_generator(generator=train_generator,
steps_per_epoch=STEP_SIZE_TRAIN,
validation_data=valid_generator,
validation_steps=STEP_SIZE_VALID,
epochs=NEW_DATA_EPOCHS,
callbacks=callback_list,
verbose=1 ).history
le = LabelEncoder()
X_xTrain.Country = le.fit_transform(X_xTrain.Country)
 X_xTrain['State'] = le.fit_transform(X_xTrain['State'])
 X_xTest.Country = le.fit_transform(X_xTest.Country)
 X_xTest['State'] = le.fit_transform(X_xTest['State'] )<categorify>
optimizer = RAdam(lr=0.00005)
 metric_list = ["accuracy"]
model = load_model('EfficientNetB5.h5', compile=False)
 model.compile(optimizer=optimizer, loss="binary_crossentropy", metrics=metric_list )
filterwarnings('ignore')
 le = LabelEncoder()
countries = X_xTrain.Country.unique()<prepare_x_and_y>
test_datagen=ImageDataGenerator(rescale=1./255.)
test_generator=test_datagen.flow_from_dataframe(
 dataframe=TEST_DF,
directory=TEST_DIR,
x_col=X_COL,
y_col=None,
batch_size=BATCH_SIZE,
seed=SEED,
shuffle=False,
class_mode=None,
preprocessing_function=preprocess_image,
target_size=(IMG_WIDTH,IMG_HEIGHT))
xout = pd.DataFrame({'ForecastId': [], 'ConfirmedCases': [], 'Fatalities': []})
 for country in countries:
states = X_xTrain.loc[X_xTrain.Country == country, :].State.unique()
for state in states:
X_xTrain_CS = X_xTrain.loc[(X_xTrain.Country == country)&(X_xTrain.State == state), ['State', 'Country', 'Date', 'ConfirmedCases', 'Fatalities']]
y1_xTrain_CS = X_xTrain_CS.loc[:, 'ConfirmedCases']
y2_xTrain_CS = X_xTrain_CS.loc[:, 'Fatalities']
X_xTrain_CS = X_xTrain_CS.loc[:, ['State', 'Country', 'Date']]
X_xTrain_CS.Country = le.fit_transform(X_xTrain_CS.Country)
 X_xTrain_CS['State'] = le.fit_transform(X_xTrain_CS['State'])
 X_xTest_CS = X_xTest.loc[(X_xTest.Country == country)&(X_xTest.State == state), ['State', 'Country', 'Date', 'ForecastId']]
X_xTest_CS_Id = X_xTest_CS.loc[:, 'ForecastId']
X_xTest_CS = X_xTest_CS.loc[:, ['State', 'Country', 'Date']]
X_xTest_CS.Country = le.fit_transform(X_xTest_CS.Country)
 X_xTest_CS['State'] = le.fit_transform(X_xTest_CS['State'])
 param = {'max_depth':range(2,6),'n_estimators':[100,500,1000]}
xgb1 =XGBRegressor()
x1=GridSearchCV(xgb1,param_grid=param,cv=3,scoring='neg_mean_squared_error')
 x1.fit(X_xTrain_CS, y1_xTrain_CS)
 y1_xpred = x1.predict(X_xTest_CS)
 xgb2 = XGBRegressor()
x2=GridSearchCV(xgb2,param_grid=param,cv=3,scoring='neg_mean_squared_error')
 x2.fit(X_xTrain_CS, y2_xTrain_CS)
 y2_xpred = x2.predict(X_xTest_CS)
 print(X_xTrain_CS.shape)
 xdata = pd.DataFrame({'ForecastId': X_xTest_CS_Id, 'ConfirmedCases': y1_xpred, 'Fatalities': y2_xpred})
 xout = pd.concat([xout, xdata], axis=0 )<save_to_csv>
if test_generator.n%BATCH_SIZE > 0:
PREDICTION_STEPS =(test_generator.n//BATCH_SIZE)+ 1
else:
PREDICTION_STEPS =(test_generator.n//BATCH_SIZE)
 print(PREDICTION_STEPS )
xout.ForecastId = xout.ForecastId.astype('int')
 xout.to_csv('submission.csv', index=False )<load_from_csv>
test_generator.reset()
preds = model.predict_generator(test_generator,
steps=PREDICTION_STEPS,
verbose=1)
 predictions = [np.argmax(pred)for pred in preds]
train_df = pd.read_csv(".. /input/covid19-global-forecasting-week-4/train.csv")
 test_df = pd.read_csv(".. /input/covid19-global-forecasting-week-4/test.csv")
 submission = pd.read_csv(".. /input/covid19-global-forecasting-week-4/submission.csv" )<drop_column>
filenames = test_generator.filenames
results = pd.DataFrame({'id_code':filenames, 'diagnosis':predictions})
 results['id_code'] = results['id_code'].map(lambda x: str(x)[:-4])
 results.astype({'diagnosis': 'int64'})
 results.to_csv('submission.csv',index=False)
 print(results.head(10))
train_df['Province_State'].fillna("",inplace = True)
 test_df['Province_State'].fillna("",inplace = True)
 train_df['Country_Region'] = train_df['Country_Region'] + ' ' + train_df['Province_State']
test_df['Country_Region'] = test_df['Country_Region'] + ' ' + test_df['Province_State']
del train_df['Province_State']
del test_df['Province_State']<feature_engineering>
t_finish = time.time()
total_time = round(( t_finish-t_start)/ 3600, 4)
 print('Kernel runtime = {} hours({} minutes)'.format(total_time, int(total_time*60)) )