Kaggle competition: APTOS 2019 Blindness Detection
<SOS> metric: QuadraticWeightedKappa Kaggle data source: aptos-2019-blindness-detection<drop_column>
nr_seed = 11
np.random.seed(nr_seed)
 tf.set_random_seed(nr_seed )
dataset = add_first_last_sale(dataset,df,['shop_id'],'shop')
 dataset = dataset.drop(['shop_first_sold','shop_last_sold'],axis=1 )<categorify>
!pip install -U '.. /input/install/efficientnet-0.0.3-py2.py3-none-any.whl'
warnings.filterwarnings("ignore")
 %matplotlib inline
%%time
shop_month_sale = create_mean_encoding_table(dataset, ['shop_id'], 'shop')
 lag_months = [1,2,3,6,12]
lag_features_list = ['shop_avg_cnt_month', 'shop_avg_revenue_month', 'shop_avg_price_month']
on_key = ['shop_id','date_block_num']
fillna = [0,0,0]
dtypes = ['float16','float32','float32']
dataset = add_lag_v3(dataset,shop_month_sale,lag_months,lag_features_list,on_key,fillna,dtypes)
 dataset = dataset.drop(['shop_avg_cnt_month', 'shop_avg_revenue_month','shop_avg_price_month'],axis=1)
 del shop_month_sale
gc.collect()<categorify>
WIDTH= 320
HEIGHT = 320
BATCH_SIZE = 32
%%time
shop_city_month_sale = create_mean_encoding_table(dataset, ['shop_city_enc'], 'shop_city')
 lag_months = [1]
lag_features_list = ['shop_city_avg_cnt_month', 'shop_city_avg_revenue_month', 'shop_city_avg_price_month']
on_key = ['shop_city_enc','date_block_num']
fillna = [0,0,0]
dtypes = ['float16','float32','float32']
dataset = add_lag_v3(dataset,shop_city_month_sale,lag_months,lag_features_list,on_key,fillna,dtypes)
 dataset = dataset.drop(['shop_city_avg_cnt_month','shop_city_avg_revenue_month','shop_city_avg_price_month'],axis=1)
 del shop_city_month_sale
gc.collect()<categorify>
new_train = pd.read_csv('.. /input/aptos2019-blindness-detection/train.csv')
 old_train = pd.read_csv('.. /input/diabetic-retinopathy-resized/trainLabels_cropped.csv')
 duplicates = pd.read_csv('.. /input/aptos-trained-weights/inconsistent.csv')
 print(new_train.shape)
 print(old_train.shape)
 print(duplicates.shape )
<categorify>
old_train = old_train[['image','level']]
old_train.columns = new_train.columns
old_train.diagnosis.value_counts()
new_train['id_code'] = '.. /input/aptos2019-blindness-detection/train_images/' + new_train['id_code'].astype(str)+ '.png'
old_train['id_code'] = '.. /input/diabetic-retinopathy-resized/resized_train/resized_train/' + old_train['id_code'].astype(str)+ '.jpeg'
train_df = old_train.copy()
val_df = new_train.copy()
%%time
shop_main_cate_month_sale = create_mean_encoding_table(dataset, ['shop_id','main_category_enc'], 'shop_main_cate')
 lag_months = [1]
lag_features_list = ['shop_main_cate_avg_cnt_month', 'shop_main_cate_avg_revenue_month', 'shop_main_cate_avg_price_month']
on_key = ['shop_id','main_category_enc','date_block_num']
fillna = [0,0,0]
dtypes = ['float16','float32','float32']
dataset = add_lag_v3(dataset,shop_main_cate_month_sale,lag_months,lag_features_list,on_key,fillna,dtypes)
 dataset = dataset.drop(['shop_main_cate_avg_cnt_month','shop_main_cate_avg_revenue_month','shop_main_cate_avg_price_month'],axis=1)
 del shop_main_cate_month_sale
gc.collect()<categorify>
def preprocess_image(image_path, width=320, height=320, new_data=False):
img = cv2.imread(image_path)
 img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
 img = circle_crop(img)
 img = cv2.resize(img,(width,height))
return img
%%time
shop_sub_cate_month_sale = create_mean_encoding_table(dataset, ['shop_id','sub_category_enc'], 'shop_sub_cate')
 lag_months = [1,2,3,6,12]
lag_features_list = ['shop_sub_cate_avg_cnt_month', 'shop_sub_cate_avg_revenue_month', 'shop_sub_cate_avg_price_month']
on_key = ['shop_id','sub_category_enc','date_block_num']
fillna = [0,0,0]
dtypes = ['float16','float32','float32']
dataset = add_lag_v3(dataset,shop_sub_cate_month_sale,lag_months,lag_features_list,on_key,fillna,dtypes)
 dataset = dataset.drop(['shop_sub_cate_avg_cnt_month','shop_sub_cate_avg_revenue_month','shop_sub_cate_avg_price_month'],axis=1)
 del shop_sub_cate_month_sale
gc.collect()<categorify>
N = val_df.shape[0]
x_val = np.empty(( N, HEIGHT, WIDTH, 3), dtype=np.uint8)
 for i, image_id in enumerate(tqdm_notebook(val_df['id_code'])) :
x_val[i, :, :, :] = preprocess_image(
 f'{image_id}',
height=HEIGHT, width=WIDTH, new_data=True
)
%%time
item_IP = pd.read_csv(items_IP_path)
 IP_label = LabelEncoder()
item_IP['IP_encode'] = IP_label.fit_transform(item_IP['Search_Key_Word'])
 dataset = dataset.merge(item_IP[['item_id','IP_encode']],on='item_id',how='left')
 dataset['IP_encode'] = dataset['IP_encode'].astype('int16')
 IPs = list(item_IP['Search_Key_Word'].unique())
 IPs.remove('Others')
 IP_trend = get_google_trends(IPs, items_IP_trend_path,'Total War')
 date_features[['year','month']] = date_features[['year','month']].astype('int64')
 IP_trend = IP_trend.merge(date_features,on=['year','month'],how='left')
 IP_trend = IP_trend.drop(['month','year'],axis=1)
 IP_trend = IP_trend.set_index('date_block_num' ).stack().reset_index()
IP_trend.columns = ['date_block_num','Search_Key_Word','google_trends']
IP_trend = IP_trend.merge(item_IP[['Search_Key_Word','IP_encode']].drop_duplicates() ,on='Search_Key_Word',how='left')
 IP_trend = IP_trend.dropna()
IP_trend = IP_trend.sort_values(by=['IP_encode','date_block_num'] )<drop_column>
y_train = pd.get_dummies(train_df['diagnosis'] ).values
y_val = pd.get_dummies(val_df['diagnosis'] ).values
print(y_train.shape)
 print(x_val.shape)
 print(y_val.shape )
%%time
lag_months = [1,2,3,6,12]
lag_features_list = ['google_trends']
on_key = ['IP_encode','date_block_num']
fillna = [0]
dtypes = ['int8']
dataset = add_lag_v3(dataset,IP_trend,lag_months,lag_features_list,on_key,fillna,dtypes)
 dataset = dataset.drop(['Search_Key_Word','google_trends'],axis=1)
 del IP_trend
gc.collect()<categorify>
y_train_multi = np.empty(y_train.shape, dtype=y_train.dtype)
 y_train_multi[:, 4] = y_train[:, 4]
for i in range(3, -1, -1):
y_train_multi[:, i] = np.logical_or(y_train[:, i], y_train_multi[:, i+1])
 y_val_multi = np.empty(y_val.shape, dtype=y_val.dtype)
 y_val_multi[:, 4] = y_val[:, 4]
for i in range(3, -1, -1):
y_val_multi[:, i] = np.logical_or(y_val[:, i], y_val_multi[:, i+1])
 print("Y_train multi: {}".format(y_train_multi.shape))
print("Y_val multi: {}".format(y_val_multi.shape))
%%time
IP_month_sale = create_mean_encoding_table(dataset, ['IP_encode'], 'IP')
 lag_months = [1]
lag_features_list = ['IP_avg_cnt_month', 'IP_avg_revenue_month', 'IP_avg_price_month']
on_key = ['IP_encode','date_block_num']
fillna = [0,0,0]
dtypes = ['float16','float32','float32']
dataset = add_lag_v3(dataset,IP_month_sale,lag_months,lag_features_list,on_key,fillna,dtypes)
 dataset = dataset.drop(['IP_avg_cnt_month', 'IP_avg_revenue_month','IP_avg_price_month'],axis=1)
 del IP_month_sale
gc.collect()<drop_column>
y_train = y_train_multi
y_val = y_val_multi
del IPs
del item_IP
del date_features
del lag_months
del lag_features_list
del on_key
del fillna
del dtypes
del IP_label
del LabelEncoder
del df
del df_test
del sample_submission
del items
del shops
gc.collect()<load_pretrained>
del new_train
del old_train
del val_df
gc.collect()
assert dataset.isnull().sum().sum() == 0
assert np.isfinite(dataset ).sum().sum() == dataset.shape[0] * dataset.shape[1]
dataset = dataset.drop(['avg_price','revenue'],axis=1)
 dataset.to_pickle(processed_data_path + 'dataset.pkl')
<load_pretrained>
class Metrics(Callback):
def on_epoch_end(self, epoch, logs={}):
X_val, y_val = self.validation_data[:2]
y_val = y_val.sum(axis=1)- 1
y_pred = self.model.predict(X_val)> 0.5
y_pred = y_pred.astype(int ).sum(axis=1)- 1
_val_kappa = cohen_kappa_score(
 y_val,
y_pred,
weights='quadratic' 
)
 self.val_kappas.append(_val_kappa)
 print(f"val_kappa: {_val_kappa:.4f}")
 if _val_kappa == max(self.val_kappas):
print("Validation Kappa has improved.Saving model.")
 self.model.save('model.h5')
 return
dataset = pd.read_pickle(processed_data_path + 'dataset.pkl' )<define_variables>
def create_datagen() :
return ImageDataGenerator(
 horizontal_flip=True,
vertical_flip=True,
zoom_range= 0.3,
brightness_range=(0.5, 2),
width_shift_range= 0.1,
height_shift_range = 0.1,
fill_mode='constant',
cval=0 
)
selected = [
'item_cnt_month',
'date_block_num',
'item_cnt_month_lag_1m','item_cnt_month_lag_2m','item_cnt_month_lag_3m',
'item_cnt_month_lag_6m','item_cnt_month_lag_12m','pair_first_sold_offset',
'item_avg_cnt_month_lag_1m','item_first_sold_offset',
'main_cate_avg_cnt_month_lag_1m','main_cate_avg_revenue_month_lag_1m',
'sub_cate_avg_revenue_month_lag_1m','sub_cate_avg_cnt_month_lag_1m',
'cate_avg_first_sold_cnt_month','main_cate_avg_first_sold_cnt_month','sub_cate_avg_first_sold_cnt_month',
'month',
'shop_avg_cnt_month_lag_1m','shop_avg_cnt_month_lag_2m','shop_avg_cnt_month_lag_3m',
'shop_avg_cnt_month_lag_6m','shop_avg_cnt_month_lag_12m','shop_city_avg_cnt_month_lag_1m',
'shop_first_sold_offset','shop_last_sold_offset',
'shop_main_cate_avg_cnt_month_lag_1m','shop_sub_cate_avg_cnt_month_lag_1m','shop_sub_cate_avg_cnt_month_lag_2m',
'shop_sub_cate_avg_cnt_month_lag_3m','shop_sub_cate_avg_cnt_month_lag_6m','shop_sub_cate_avg_cnt_month_lag_12m',
'IP_avg_cnt_month_lag_1m','IP_avg_revenue_month_lag_1m',
'google_trends_lag_1m','google_trends_lag_2m','google_trends_lag_3m',
'google_trends_lag_6m','google_trends_lag_12m',
]
dataset = dataset[selected]<split>
fig, ax = plt.subplots(1, 10, figsize=(20, 10))
ax = ax.ravel()
img = x_val[0].reshape(1,x_val[0].shape[0],x_val[0].shape[1], x_val[0].shape[2])
 ax[0].imshow(img[0].astype('uint8'))
ax[1].imshow(next(ImageDataGenerator().flow(img)) [0].astype('uint8'))
ax[2].imshow(next(ImageDataGenerator(horizontal_flip=True, fill_mode='constant', cval=0 ).flow(img)) [0].astype('uint8'))
ax[3].imshow(next(ImageDataGenerator(vertical_flip=True,fill_mode='constant', cval=0 ).flow(img)) [0].astype('uint8'))
ax[4].imshow(next(ImageDataGenerator(rotation_range=360, fill_mode='constant', cval=0 ).flow(img)) [0].astype('uint8'))
ax[5].imshow(next(ImageDataGenerator(zoom_range=(0.65,1), fill_mode='constant', cval=0 ).flow(img)) [0].astype('uint8'))
ax[6].imshow(next(ImageDataGenerator(height_shift_range=0.15, fill_mode='constant', cval=0 ).flow(img)) [0].astype('uint8'))
ax[7].imshow(next(ImageDataGenerator(width_shift_range=0.15, fill_mode='constant', cval=0 ).flow(img)) [0].astype('uint8'))
ax[8].imshow(next(ImageDataGenerator(brightness_range=(0.5, 2), fill_mode='constant', cval=0 ).flow(img)) [0].astype('uint8'))
ax[9].imshow(next(ImageDataGenerator(horizontal_flip=True,
vertical_flip=True,
zoom_range=0.3,
brightness_range=(0.5, 2),
fill_mode='constant',cval=0 ).flow(img)) [0].astype('uint8'))
train = dataset[dataset['date_block_num'].between(12,32)]
validate = dataset[dataset['date_block_num']==33]
test = dataset[dataset['date_block_num']==34]
del dataset
gc.collect()<prepare_x_and_y>
efficientnetb3 = EfficientNetB3(
 weights=None,
input_shape=(HEIGHT,WIDTH,3),
include_top=False 
)
 efficientnetb3.load_weights(".. /input/efficientnet-keras-weights-b0b5/efficientnet-b3_imagenet_1000_notop.h5" )
X_train = train.drop(['item_cnt_month','date_block_num'],axis=1)
 y_train = train['item_cnt_month']
X_validate = validate.drop(['item_cnt_month','date_block_num'],axis=1)
 y_validate = validate['item_cnt_month']
X_test = test.drop(['item_cnt_month','date_block_num'],axis=1)
 del train
del test
del validate
gc.collect()<count_missing_values>
def build_model() :
model = Sequential()
model.add(efficientnetb3)
 model.add(layers.GlobalAveragePooling2D())
 model.add(layers.Dropout(0.5))
model.add(layers.BatchNormalization())
 model.add(layers.Dense(5, activation='sigmoid'))
model.compile(
 loss='binary_crossentropy',
optimizer=Adam(lr=1e-4,decay=1e-6),
metrics=['accuracy'] 
)
 return model
print(f'training shape: X {X_train.shape}, y {y_train.shape}')
 print(f'validation shape: X {X_validate.shape}, y {y_validate.shape}')
 print(f'testing shape: X {X_test.shape}')
 print(f'training missing: X {X_train.isnull().sum().sum() }, y {y_train.isnull().sum().sum() }')
 print(f'validation missing: X {X_validate.isnull().sum().sum() }, y {y_validate.isnull().sum().sum() }')
 print(f'testing missing: X {X_test.isnull().sum().sum() }' )<choose_model_class>
bucket_num = 8
div = round(train_df.shape[0]/bucket_num )
def create_model() :
model = xgb.XGBRegressor(eta=0.01, max_depth=10,n_estimators=2000,
colsample_bytree=0.5,subsample=0.8,
gamma=2, reg_alpha=0, reg_lambda=2, min_child_weight=300,
tree_method='gpu_hist', gpu_id=0, max_bin=2048,
n_jobs=-1)
 return model<train_model>
df_init = {
'val_loss': [0.0],
'val_acc': [0.0],
'loss': [0.0],
'acc': [0.0],
'bucket': [0.0]
}
results = pd.DataFrame(df_init )
%%time
model = create_model()
model.fit(X_train,y_train,eval_set=[(X_train,y_train),(X_validate,y_validate)],eval_metric='rmse',early_stopping_rounds=10 )<load_pretrained>
epochs = [5,5,5,5,5,5,5,5]
kappa_metrics = Metrics()
kappa_metrics.val_kappas = []
learn_control = ReduceLROnPlateau(monitor='val_loss',
patience=5,
verbose=1,
factor=.5,
min_lr=1e-7)
 checkpoint = ModelCheckpoint('val_model.h5', monitor='val_loss', verbose=1, save_best_only=True, mode='min' )
with open(model_path + 'model.pkl', 'wb')as f:
pickle.dump(model, f )<predict_on_test>
results = results.iloc[1:]
results['kappa'] = kappa_metrics.val_kappas
results = results.reset_index()
results = results.rename(index=str, columns={"index": "epoch"})
 results
pred_test = model.predict(X_test)
 output = pd.DataFrame({'item_cnt_month':pred_test})
 output.index.name = 'ID'<save_to_csv>
results[['loss', 'val_loss']].plot()
results[['acc', 'val_acc']].plot()
results[['kappa']].plot()
results.to_csv('model_results.csv',index=False )
output = output.clip(0,20)
 output.to_csv('./submission.csv' )<define_variables>
model.load_weights('val_model.h5' )
sys.path.append(os.path.abspath('.. /input/efficientnet/efficientnet-master/efficientnet-master/'))
TRAIN_DATA_PATH = ".. /input/aptos2019-blindness-detection"
TEST_DATA_PATH = ".. /input/aptos2019-blindness-detection"
PREPROCESSED_IMAGE_PATH = "./preprocessed"
MODEL_PATH = "./models"
MODEL_FILE_NAME = "effnet_b3_single.h5"
TRAIN_CSV_FILE_PATH = os.path.join(TRAIN_DATA_PATH, "train.csv")
 TRAIN_IMAGE_FILE_PATH = ".. /input/aptos2019-blindness-detection/train_images"
TEST_CSV_FILE_PATH = ".. /input/aptos2019-blindness-detection/test.csv"
TEST_IMAGE_FILE_PATH = ".. /input/aptos2019-blindness-detection/test_images"

 CHECK_KERNEL_VALID = False
IMG_WIDTH = 456
IMG_HEIGHT = 456
IMG_CHANNELS = 3
BATCH_SIZE = 4
NUM_FOLDS = 6
EPOCHS = 30
if CHECK_KERNEL_VALID:
EPOCHS = 1
GENERATE_WEIGHTS = True
TRAIN_OVER_PRETRAINED = True
RESCALE_DN = 128.0<load_from_csv>
x_train, x_val, y_train, y_val = train_test_split(
 x_val, y_val,
test_size=0.2,
random_state=nr_seed 
)
 gc.collect()
df_train = pd.read_csv(TRAIN_CSV_FILE_PATH)
 df_train['id_code'] = df_train['id_code'] + ".png"
df_test = pd.read_csv(TEST_CSV_FILE_PATH)
 df_train.head()<predict_on_test>
data_generator = create_datagen().flow(x_train, y_train, batch_size=BATCH_SIZE )
def get_preds_and_labels(model, generator):

 preds = []
labels = []
for _ in range(int(np.ceil(generator.samples / BATCH_SIZE))):
x, y = next(generator)
 preds.append(model.predict(x))
labels.append(y)
 return np.concatenate(preds ).ravel() , np.concatenate(labels ).ravel() 
<train_model>
history = model.fit_generator(
 data_generator,
steps_per_epoch=x_train.shape[0] / BATCH_SIZE,
epochs=20,
validation_data=(x_val, y_val),
callbacks=[kappa_metrics,learn_control,checkpoint]
)
class Metrics(Callback):

 def __init__(self, model, val_generator, model_save_filepath):
self.model = model
self.val_generator = val_generator
self.model_save_filepath = model_save_filepath
def on_train_begin(self, logs={}):

 self.val_kappas = []
def on_epoch_end(self, epoch, logs={}):

 y_pred, labels = get_preds_and_labels(self.model, self.val_generator)
 y_pred = np.rint(y_pred ).astype(np.uint8 ).clip(0, 4)
 _val_kappa = cohen_kappa_score(labels, y_pred, weights='quadratic')
 self.val_kappas.append(_val_kappa)
 print(f"val_kappa: {round(_val_kappa, 4)}")
 if _val_kappa == max(self.val_kappas):
print("Validation Kappa has improved.Saving model.")
 self.model.save(self.model_save_filepath)
 return<choose_model_class>
model.load_weights('val_model.h5')
 pred_val = model.predict(x_val )
def get_callbacks(model, val_generator, model_save_filepath):
es = EarlyStopping(monitor='val_loss', mode='auto', verbose=1, patience=12)
 lr = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=4,
verbose=1, mode='auto', min_delta=0.0001)
 km = Metrics(model, val_generator, model_save_filepath)
 return [es, lr, km]<define_variables>
def compute_score_inv(threshold):
y1 = pred_val > threshold
y1 = y1.astype(int ).sum(axis=1)- 1
y2 = y_val.sum(axis=1)- 1
score = cohen_kappa_score(y1, y2, weights='quadratic')
 return 1 - score
simplex = scipy.optimize.minimize(compute_score_inv, 0.5, method='nelder-mead')
 best_threshold = simplex['x'][0]
print(best_threshold)
 gc.collect()