Kaggle competition: APTOS 2019 Blindness Detection
group = train_new.groupby(["date_block_num", "shop_id", "big_category_id"] ).agg({"item_cnt_month": ["mean"]} ).reset_index()
group.columns = ["date_block_num", "shop_id", "big_category_id", "shop_big_category_item_cnt_mean"]
train_new = train_new.merge(group, on=["date_block_num", "shop_id", "big_category_id"])
 train_new = lag_feature(train_new, [1], "shop_big_category_item_cnt_mean")
 train_new.drop("shop_big_category_item_cnt_mean", axis=1, inplace=True)
 train_new.fillna(0, inplace=True )<merge>
BatchNormalization, Input, Conv2D, GlobalAveragePooling2D,concatenate,Concatenate)
group = train_new.groupby(["date_block_num", "shop_id", "sub_category_id"] ).agg({"item_cnt_month": ["mean"]} ).reset_index()
group.columns = ["date_block_num", "shop_id", "sub_category_id", "shop_sub_category_item_cnt_mean"]
train_new = train_new.merge(group, on=["date_block_num", "shop_id", "sub_category_id"])
 train_new = lag_feature(train_new, [1], "shop_sub_category_item_cnt_mean")
 train_new.drop("shop_sub_category_item_cnt_mean", axis=1, inplace=True)
 train_new.fillna(0, inplace=True )<feature_engineering>
WORKERS = 2
CHANNEL = 3
warnings.filterwarnings("ignore")
 SIZE = 300
NUM_CLASSES = 5
train_new["month"] = train_new["date_block_num"] % 12<categorify>
df_train = pd.read_csv('.. /input/aptos2019-blindness-detection/train.csv')
 df_test = pd.read_csv('.. /input/aptos2019-blindness-detection/test.csv' )
holidays = [13, 8, 11, 8, 10, 11, 8, 9, 9, 8, 10, 9,
12, 8, 10, 4, 11, 10, 8, 10, 8, 8, 12, 8,
13, 9, 9, 8, 11, 9, 8, 10, 8, 9, 11]
holidays_dict = dict(zip(list(range(35)) , holidays))
print(holidays_dict)
 train_new["holidays_cnt"] = train_new["date_block_num"].map(holidays_dict )<define_variables>
y = to_categorical(y, num_classes=NUM_CLASSES)
 train_x, valid_x, train_y, valid_y = train_test_split(x, y, test_size=0.15,
stratify=y, random_state=8)
 print(train_x.shape)
 print(train_y.shape)
 print(valid_x.shape)
 print(valid_y.shape )
rates = [
0.033064, 0.033135, 0.032450, 0.031901, 0.031923, 0.030951, 0.030517, 0.030310, 0.030674, 0.031174, 0.030592, 0.030425,
0.029695, 0.028373, 0.027628, 0.028044, 0.028638, 0.029076, 0.028831, 0.027667, 0.026349, 0.024501, 0.021618, 0.017868,
0.015704, 0.015517, 0.016594, 0.018804, 0.019731, 0.018311, 0.017496, 0.015301, 0.014936, 0.015841, 0.015365
]
rates_dict = dict(zip(list(range(35)) , rates))
train_new["dollar_ruble_rate"] = train_new["date_block_num"].map(rates_dict )<categorify>
class My_Generator(Sequence):
def __init__(self, image_filenames, labels,
batch_size, is_train=True,
mix=False, augment=False):
self.image_filenames, self.labels = image_filenames, labels
self.batch_size = batch_size
self.is_train = is_train
self.is_augment = augment
if(self.is_train):
self.on_epoch_end()
self.is_mix = mix
def __len__(self):
return int(np.ceil(len(self.image_filenames)/ float(self.batch_size)))
 def __getitem__(self, idx):
batch_x = self.image_filenames[idx * self.batch_size:(idx + 1)* self.batch_size]
batch_y = self.labels[idx * self.batch_size:(idx + 1)* self.batch_size]
if(self.is_train):
return self.train_generate(batch_x, batch_y)
 return self.valid_generate(batch_x, batch_y)
 def on_epoch_end(self):
if(self.is_train):
self.image_filenames, self.labels = shuffle(self.image_filenames, self.labels)
 else:
pass
def mix_up(self, x, y):
lam = np.random.beta(0.2, 0.4)
 ori_index = np.arange(int(len(x)))
 index_array = np.arange(int(len(x)))
 np.random.shuffle(index_array)
 mixed_x = lam * x[ori_index] +(1 - lam)* x[index_array]
mixed_y = lam * y[ori_index] +(1 - lam)* y[index_array]
return mixed_x, mixed_y
def train_generate(self, batch_x, batch_y):
batch_images = []
for(sample, label)in zip(batch_x, batch_y):
img = cv2.imread('.. /input/aptos2019-blindness-detection/train_images/'+sample+'.png')
 img = cv2.resize(img,(SIZE, SIZE))
if(self.is_augment):
img = seq.augment_image(img)
 batch_images.append(img)
 batch_images = np.array(batch_images, np.float32)/ 255
batch_y = np.array(batch_y, np.float32)
 if(self.is_mix):
batch_images, batch_y = self.mix_up(batch_images, batch_y)
 return batch_images, batch_y
def valid_generate(self, batch_x, batch_y):
batch_images = []
for(sample, label)in zip(batch_x, batch_y):
img = cv2.imread('.. /input/aptos2019-blindness-detection/train_images/'+sample+'.png')
 img = cv2.resize(img,(SIZE, SIZE))
batch_images.append(img)
 batch_images = np.array(batch_images, np.float32)/ 255
batch_y = np.array(batch_y, np.float32)
 return batch_images, batch_y
days = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
days_dict = dict(zip(list(range(13)) , rates))
train_new["days_cnt"] = train_new["month"].map(days_dict )<load_from_csv>
def create_model(input_shape, n_out):
input_tensor = Input(shape=input_shape)
 base_model = DenseNet121(include_top=False,
weights=None,
input_tensor=input_tensor)
 base_model.load_weights(".. /input/densenet-keras/DenseNet-BC-121-32-no-top.h5")
 x = GlobalAveragePooling2D()(base_model.output)
 x = Dropout(0.5 )(x)
 x = Dense(1024, activation='relu' )(x)
 x = Dropout(0.5 )(x)
 final_output = Dense(n_out, activation='softmax', name='final_output' )(x)
 model = Model(input_tensor, final_output)
 return model
test_sort = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/test.csv')
 test_sort.drop("ID", axis=1, inplace = True)
 train = downcast(train_new.query('date_block_num < 31 and date_block_num > 5'))
val = downcast(train_new.query('date_block_num > 30 and date_block_num < 34'))
test = downcast(test_sort.merge(train_new.query('date_block_num == 34'), on = ["shop_id", "item_id"], how="left"))<categorify>
EarlyStopping, ReduceLROnPlateau,CSVLogger)
 epochs = 30; batch_size = 32
checkpoint = ModelCheckpoint('.. /working/densenet_.h5', monitor='val_loss', verbose=1,
save_best_only=True, mode='min', save_weights_only = True)
 reduceLROnPlat = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=4,
verbose=1, mode='auto', epsilon=0.0001)
 early = EarlyStopping(monitor="val_loss",
mode="min",
patience=9)
 csv_logger = CSVLogger(filename='.. /working/training_log.csv',
separator=',',
append=True)
 train_generator = My_Generator(train_x, train_y, 128, is_train=True)
 train_mixup = My_Generator(train_x, train_y, batch_size, is_train=True, mix=False, augment=True)
 valid_generator = My_Generator(valid_x, valid_y, batch_size, is_train=False)
 model = create_model(
 input_shape=(SIZE,SIZE,3),
n_out=NUM_CLASSES )
train = downcast(train)
 val = downcast(val)
 test = downcast(test )<prepare_x_and_y>
def kappa_loss(y_true, y_pred, y_pow=2, eps=1e-12, N=5, bsize=32, name='kappa'):

 with tf.name_scope(name):
y_true = tf.to_float(y_true)
 repeat_op = tf.to_float(tf.tile(tf.reshape(tf.range(0, N), [N, 1]), [1, N]))
repeat_op_sq = tf.square(( repeat_op - tf.transpose(repeat_op)))
 weights = repeat_op_sq / tf.to_float(( N - 1)** 2)
 pred_ = y_pred ** y_pow
try:
pred_norm = pred_ /(eps + tf.reshape(tf.reduce_sum(pred_, 1), [-1, 1]))
except Exception:
pred_norm = pred_ /(eps + tf.reshape(tf.reduce_sum(pred_, 1), [bsize, 1]))
hist_rater_a = tf.reduce_sum(pred_norm, 0)
 hist_rater_b = tf.reduce_sum(y_true, 0)
 conf_mat = tf.matmul(tf.transpose(pred_norm), y_true)
 nom = tf.reduce_sum(weights * conf_mat)
 denom = tf.reduce_sum(weights * tf.matmul(
 tf.reshape(hist_rater_a, [N, 1]), tf.reshape(hist_rater_b, [1, N])) /
tf.to_float(bsize))
return nom*0.5 /(denom + eps)+ categorical_crossentropy(y_true, y_pred)*0.5
X_train = train.drop("item_cnt_month", axis = 1 ).reset_index(drop=True)
 X_val = val.drop("item_cnt_month", axis = 1 ).reset_index(drop=True)
 y_train = train[["item_cnt_month"]].reset_index(drop=True)
 y_val = val[["item_cnt_month"]].reset_index(drop=True)
 X_test = test.drop("item_cnt_month", axis = 1 ).reset_index(drop=True )<categorify>
class QWKEvaluation(Callback):
def __init__(self, validation_data=() , batch_size=64, interval=1):
super(Callback, self ).__init__()
self.interval = interval
self.batch_size = batch_size
self.valid_generator, self.y_val = validation_data
self.history = []
def on_epoch_end(self, epoch, logs={}):
if epoch % self.interval == 0:
y_pred = self.model.predict_generator(generator=self.valid_generator,
steps=np.ceil(float(len(self.y_val)) / float(self.batch_size)) ,
workers=1, use_multiprocessing=False,
verbose=1)
 def flatten(y):
return np.argmax(y, axis=1 ).reshape(-1)
 score = cohen_kappa_score(flatten(self.y_val),
flatten(y_pred),
labels=[0,1,2,3,4],
weights='quadratic')
 print("
epoch: %d - QWK_score: %.6f
" %(epoch+1, score))
self.history.append(score)
 if score >= max(self.history):
print('saving checkpoint: ', score)
 self.model.save('.. /working/densenet_bestqwk.h5')
 qwk = QWKEvaluation(validation_data=(valid_generator, valid_y),
batch_size=batch_size, interval=1 )
X_train = downcast(X_train)
 X_val = downcast(X_val)
 y_train = downcast(y_train)
 y_val = downcast(y_val)
 X_test = downcast(X_test )<drop_column>
for layer in model.layers:
layer.trainable = False
for i in range(-3,0):
model.layers[i].trainable = True
model.compile(
 loss='categorical_crossentropy',
optimizer=Adam(1e-3))
model.fit_generator(
 train_generator,
steps_per_epoch=np.ceil(float(len(train_y)) / float(128)) ,
epochs=2,
workers=WORKERS, use_multiprocessing=True,
verbose=1,
callbacks=[qwk] )
del train_new
del train
del test
del val
del items
del shops
del item_categories<set_options>
for layer in model.layers:
layer.trainable = True
callbacks_list = [checkpoint, csv_logger, reduceLROnPlat, early, qwk]
model.compile(loss='categorical_crossentropy',
optimizer=Adam(lr=1e-4))
model.fit_generator(
 train_mixup,
steps_per_epoch=np.ceil(float(len(train_x)) / float(batch_size)) ,
validation_data=valid_generator,
validation_steps=np.ceil(float(len(valid_x)) / float(batch_size)) ,
epochs=epochs,
verbose=1,
workers=1, use_multiprocessing=False,
callbacks=callbacks_list )
gc.collect()<define_variables>
submit = pd.read_csv('.. /input/aptos2019-blindness-detection/sample_submission.csv')
 model.load_weights('.. /working/densenet_bestqwk.h5')
 predicted = []
xgb_features = ['date_block_num',
'shop_id',
'item_id',
'item_category_id',
'big_category_id',
'sub_category_id',
'city_id',
'shop_category_id',
'item_cnt_month_lag_1',
'item_cnt_month_lag_2',
'item_cnt_month_lag_3',
'date_block_item_cnt_mean_lag_1',
'shop_item_cnt_mean_lag_1',
'shop_item_cnt_mean_lag_2',
'shop_item_cnt_mean_lag_3',
'item_item_cnt_mean_lag_1',
'item_category_item_cnt_mean_lag_1',
'big_category_item_cnt_mean_lag_1',
'sub_category_item_cnt_mean_lag_1',
'shop_category_item_cnt_mean_lag_1',
'item_category_item_item_cnt_mean_lag_1',
'item_category_item_item_cnt_mean_lag_2',
'shop_item_category_item_cnt_mean_lag_1',
'shop_item_category_item_cnt_mean_lag_2',
'shop_big_category_item_cnt_mean_lag_1',
'shop_sub_category_item_cnt_mean_lag_1',
'month',
'holidays_cnt',
'dollar_ruble_rate',
'days_cnt']<split>
for i, name in tqdm(enumerate(submit['id_code'])) :
path = os.path.join('.. /input/aptos2019-blindness-detection/train_images/', name+'.png')
 image = cv2.imread(path)
 image = cv2.resize(image,(SIZE, SIZE))
X = np.array(( image[np.newaxis])/255)
 score_predict=(( model.predict(X ).ravel() *model.predict(X[:, ::-1, :, :] ).ravel() *model.predict(X[:, ::-1, ::-1, :] ).ravel() *model.predict(X[:, :, ::-1, :] ).ravel())**0.25 ).tolist()
label_predict = np.argmax(score_predict)
 predicted.append(str(label_predict))
<train_model><EOS> 
submit['diagnosis'] = predicted
submit.to_csv('submission.csv', index=False)
 submit.head()