Kaggle competition: APTOS 2019 Blindness Detection
<SOS> metric: QuadraticWeightedKappa Kaggle data source: aptos-2019-blindness-detection<drop_column>
%matplotlib inline
grid = grid[grid['date_block_num'] > 11]
grid = grid.fillna(0 )<feature_engineering>
TRAINING = True
dates_train = sales.loc[:, ['date', 'date_block_num']].drop_duplicates()
dates_train = dates_train.reset_index(drop = True)
 dates_test = dates_train.loc[dates_train.loc[:, 'date_block_num'] == 34 - 12]
dates_test = dates_test.reset_index(drop=True)
 dates_test.loc[:,'date_block_num'] = 34
dates_test.loc[:, 'date'] = dates_test.loc[:, 'date'] + pd.DateOffset(years = 1)
 dates_all = pd.concat([dates_train, dates_test])
 dates_all.loc[:, 'dow'] = dates_all.loc[:, 'date'].dt.dayofweek
dates_all.loc[:, 'year'] = dates_all.loc[:, 'date'].dt.year
dates_all.loc[:, 'month'] = dates_all.loc[:, 'date'].dt.month
dates_all = pd.get_dummies(dates_all, columns = ['dow'])
 dow_col = ['dow_' + str(x)for x in range(7)]
date_features = dates_all.groupby(['year', 'month', 'date_block_num'])[dow_col].agg('sum' ).reset_index()
date_features.loc[:, 'days_of_month'] = date_features.loc[:, dow_col].sum(axis=1)
 date_features.loc[:, 'year'] = date_features.loc[:, 'year'] - 2013
date_features = date_features.loc[:, ['month', 'date_block_num']]<merge>
test_df = pd.read_csv('.. /input/aptos2019-blindness-detection/test.csv')
 print(test_df.shape)
 if TRAINING:
train_df = pd.read_csv('.. /input/aptos2019-blindness-detection/train.csv')
 print(train_df.shape)
 train_df.head()
grid = grid.merge(date_features, on = 'date_block_num', how = 'left' )<drop_column>
del dates_train
del dates_test
del dates_all
del dow_col
del date_features
gc.collect()<categorify>
IMG_SIZE = 224
NB_CHANNELS = 3
grid['category_shop_inter'] = grid['item_category_id'].astype(str)+ '_' + grid['shop_id'].astype(str)
 grid.loc[:, 'category_shop_inter'] = LabelEncoder().fit_transform(grid.loc[:, 'category_shop_inter'].values )<prepare_x_and_y>
def get_pad_width(im, new_shape, is_rgb=True):
pad_diff = new_shape - im.shape[0], new_shape - im.shape[1]
t, b = math.floor(pad_diff[0]/2), math.ceil(pad_diff[0]/2)
 l, r = math.floor(pad_diff[1]/2), math.ceil(pad_diff[1]/2)
 if is_rgb:
pad_width =(( t,b),(l,r),(0, 0))
else:
pad_width =(( t,b),(l,r))
return pad_width
def standardize(x):
x = x.astype(np.float32)
 x = x / np.max(x)
 return(x - np.mean(x)) /(np.std(x))

 def normalize(img):
img =(( img - np.min(img)) /(np.max(img)- np.min(img)))* 255
return img.astype(np.uint8)

def crop_image(img, tol=10):
def crop_image_1(img):
mask = img > tol
return img[np.ix_(mask.any(1), mask.any(0)) ]
if img.ndim == 2:
return crop_image_1(img)
 elif img.ndim == 3:
try:
img_cpy = img.copy()
h, w, _ = img.shape
img1 = cv2.resize(crop_image_1(img[:, :, 0]),(w, h))
img2 = cv2.resize(crop_image_1(img[:, :, 1]),(w, h))
img3 = cv2.resize(crop_image_1(img[:, :, 2]),(w, h))
img[:,:,0] = img1
img[:,:,1] = img2
img[:,:,2] = img3
except:
return img_cpy
return img

 def preprocess_image(im):
im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)
 im = normalize(im)
 im = crop_image(im)
 im = cv2.resize(im,(IMG_SIZE, IMG_SIZE))
im = cv2.addWeighted(im, 4, cv2.GaussianBlur(im,(0, 0), IMG_SIZE / 10), -4, 128)
 return im.astype(np.uint8 )
cols_to_drop = ['target', 'date_block_num']
def train_val_test_split(df):
dates = df['date_block_num']
last_block = dates.max()
print('Test `date_block_num` is %d' % last_block)
 print('Validation `date_block_num` is %d' %(last_block - 1))
print('Train `date_block_num` is < %d' %(last_block - 1))
print('------------------------------')
 X_train = df.loc[dates < last_block - 1].drop(cols_to_drop, axis = 1)
 X_val = df.loc[dates == last_block - 1].drop(cols_to_drop, axis = 1)
 X_test = df.loc[dates == last_block].drop(cols_to_drop, axis = 1)
 y_train = df.loc[dates < last_block - 1, 'target'].values
y_val = df.loc[dates == last_block - 1, 'target'].values
print('X_train shape: ', X_train.shape)
 print('y_train shape: ', y_train.shape)
 print('------------------------------')
 print('X_val shape: ', X_val.shape)
 print('y_val shape: ', y_val.shape)
 print('------------------------------')
 print('X_test shape: ', X_test.shape)
 print('------------------------------')
 return(X_train, y_train, X_val, y_val, X_test)
 def rmse(y, y_hat):
return np.sqrt(mean_squared_error(y, y_hat))
def create_lgbm_model(X_train, y_train, X_val, y_val, params, cat_feats):
n_estimators = 8000
d_train = lgb.Dataset(X_train, y_train)
 d_valid = lgb.Dataset(X_val, y_val)
 watchlist = [d_train, d_valid]
evals_result = {}
model = lgb.train(params,
d_train,
n_estimators,
valid_sets = watchlist,
evals_result = evals_result,
early_stopping_rounds = 50,
verbose_eval = 0,
categorical_feature = cat_feats, 
)
 lgb.plot_metric(evals_result)
 return model
def evaluate_model(model, X_train, y_train, X_val, y_val):
y_hat = model.predict(X_train)
 print('Training error;', rmse(y_train, y_hat))
y_val_hat = model.predict(X_val)
 print('Validation error:', rmse(y_val, y_val_hat))<data_type_conversions>
if TRAINING:
N = train_df.shape[0]
x_train = np.empty(( N, 224, 224, 3), dtype=np.uint8)
 for i, image_id in enumerate(tqdm(train_df['id_code'])) :
x_train[i, :, :, :] = preprocess_image(cv2.imread(
 f'.. /input/aptos2019-blindness-detection/train_images/{image_id}.png' 
))
categorical_features = ['shop_id', 'item_category_id', 'month', 'category_shop_inter']
for col in categorical_features:
grid.loc[:, col] = grid[col].astype('category' )<split>
N = test_df.shape[0]
x_test = np.empty(( N, 224, 224, 3), dtype=np.uint8)
 for i, image_id in enumerate(tqdm(test_df['id_code'])) :
x_test[i, :, :, :] = preprocess_image(cv2.imread(
 f'.. /input/aptos2019-blindness-detection/test_images/{image_id}.png' 
))
X_train, y_train, X_val, y_val, X_test = train_val_test_split(grid )<train_model>
if TRAINING:
y_train = pd.get_dummies(train_df['diagnosis'] ).values
print(x_train.shape)
 print(y_train.shape)
 print(x_test.shape )
start = time.time()
params = {
'metric': 'rmse',
'objective': 'mse',
'verbose': 0,
'learning_rate': 0.1,
'num_leaves': 31,
'min_data_in_leaf': 20 ,
'max_depth': -1,
'save_binary': True,
'bagging_fraction': 0.8,
'bagging_freq': 1,
'bagging_seed': 2**7,
'feature_fraction': 0.8,
}
lgbm_model = create_lgbm_model(X_train, y_train, X_val, y_val, params, categorical_features)
 print('it tooks: ', round(time.time() - start), 'seconds' )<train_model>
if TRAINING:
x_train, x_val, y_train, y_val = train_test_split(
 x_train, y_train,
test_size=0.2
)
start = time.time()
evaluate_model(lgbm_model, X_train, y_train, X_val, y_val)
 print('it tooks: ', round(time.time() - start), 'seconds' )<save_to_csv>
y_test_pred = lgbm_model.predict(X_test ).clip(0, 20)
 submission = pd.DataFrame({
"ID": test.ID,
"item_cnt_month": y_test_pred
})
 submission.to_csv('lgbm_submission.csv', index = False )<load_from_csv>
items = pd.read_csv('.. /input/items.csv')
 shops = pd.read_csv('.. /input/shops.csv')
 cats = pd.read_csv('.. /input/item_categories.csv')
 train = pd.read_csv('.. /input/sales_train.csv')
 test = pd.read_csv('.. /input/test.csv' ).set_index('ID' )<filter>
train = train[train.item_price<100000]
train = train[train.item_cnt_day<1001]<feature_engineering>
median = train[(train.shop_id==32)&(train.item_id==2973)&(train.date_block_num==4)&(train.item_price>0)].item_price.median()
train.loc[train.item_price<0, 'item_price'] = median<feature_engineering>
train.loc[train.shop_id == 0, 'shop_id'] = 57
test.loc[test.shop_id == 0, 'shop_id'] = 57
train.loc[train.shop_id == 1, 'shop_id'] = 58
test.loc[test.shop_id == 1, 'shop_id'] = 58
train.loc[train.shop_id == 10, 'shop_id'] = 11
test.loc[test.shop_id == 10, 'shop_id'] = 11<categorify>
if TRAINING:
indexes = np.random.permutation(len(x_train))
x_train = x_train[indexes]
y_train = y_train[indexes]
shops.loc[shops.shop_name == 'Сергиев Посад ТЦ "7Я"', 'shop_name'] = 'СергиевПосад ТЦ "7Я"'
shops['city'] = shops['shop_name'].str.split(' ' ).map(lambda x: x[0])
 shops.loc[shops.city == '!Якутск', 'city'] = 'Якутск'
shops['city_code'] = LabelEncoder().fit_transform(shops['city'])
 shops = shops[['shop_id','city_code']]
cats['split'] = cats['item_category_name'].str.split('-')
 cats['type'] = cats['split'].map(lambda x: x[0].strip())
 cats['type_code'] = LabelEncoder().fit_transform(cats['type'])
 cats['subtype'] = cats['split'].map(lambda x: x[1].strip() if len(x)> 1 else x[0].strip())
 cats['subtype_code'] = LabelEncoder().fit_transform(cats['subtype'])
 cats = cats[['item_category_id','type_code', 'subtype_code']]
items.drop(['item_name'], axis=1, inplace=True )<concatenate>
if TRAINING:
for i in range(3, -1, -1):
y_train[:, i] = np.logical_or(y_train[:, i], y_train[:, i + 1])
 y_val[:, i] = np.logical_or(y_val[:, i], y_val[:, i + 1])
 print("Multilabel version:", y_train.sum(axis=0))
len(list(set(test.item_id)- set(test.item_id ).intersection(set(train.item_id)))) , len(list(set(test.item_id))), len(test )<data_type_conversions>
if TRAINING:
for i in tqdm(range(len(x_val))):
x_val[i] = cv2.resize(x_val[i, 20: -20, 20: -20, :],(IMG_SIZE, IMG_SIZE))
for i in tqdm(range(len(x_train))):
x_train[i] = cv2.resize(x_train[i, 20: -20, 20: -20, :],(IMG_SIZE, IMG_SIZE))
ts = time.time()
matrix = []
cols = ['date_block_num','shop_id','item_id']
for i in range(34):
sales = train[train.date_block_num==i]
matrix.append(np.array(list(product([i], sales.shop_id.unique() , sales.item_id.unique())) , dtype='int16'))
matrix = pd.DataFrame(np.vstack(matrix), columns=cols)
 matrix['date_block_num'] = matrix['date_block_num'].astype(np.int8)
 matrix['shop_id'] = matrix['shop_id'].astype(np.int8)
 matrix['item_id'] = matrix['item_id'].astype(np.int16)
 matrix.sort_values(cols,inplace=True)
 time.time() - ts<feature_engineering>
class MixupGenerator() :
def __init__(self, X_train, y_train, seq=None, batch_size=32, alpha=0.2, shuffle=True, datagen=None):
self.X_train = X_train
self.y_train = y_train
self.batch_size = batch_size
self.alpha = alpha
self.shuffle = shuffle
self.sample_num = len(X_train)
 self.datagen = datagen
self.seq = seq
def __call__(self):
while True:
indexes = self.__get_exploration_order()
itr_num = int(len(indexes)//(self.batch_size * 2))
for i in range(itr_num):
batch_ids = indexes[i * self.batch_size * 2:(i + 1)* self.batch_size * 2]
X, y = self.__data_generation(batch_ids)
 yield X, y
def __get_exploration_order(self):
indexes = np.arange(self.sample_num)
 if self.shuffle:
np.random.shuffle(indexes)
 return indexes
def __data_generation(self, batch_ids):
_, h, w, c = self.X_train.shape
l = np.random.beta(self.alpha, self.alpha, self.batch_size)
 X_l = l.reshape(self.batch_size, 1, 1, 1)
 y_l = l.reshape(self.batch_size, 1)
 X1 = self.X_train[batch_ids[:self.batch_size]]
X2 = self.X_train[batch_ids[self.batch_size:]]
X = X1 * X_l + X2 *(1 - X_l)
 if self.datagen:
for i in range(self.batch_size):
X[i] = self.datagen.random_transform(X[i])
 X[i] = self.datagen.standardize(X[i])
 if self.seq:
X = X.astype(np.float32)
 X /= np.max(X)
 X *= 255
X = X.astype(np.uint8)
 X = self.seq.augment_images(X)
 if isinstance(self.y_train, list):
y = []
for y_train_ in self.y_train:
y1 = y_train_[batch_ids[:self.batch_size]]
y2 = y_train_[batch_ids[self.batch_size:]]
y.append(y1 * y_l + y2 *(1 - y_l))
else:
y1 = self.y_train[batch_ids[:self.batch_size]]
y2 = self.y_train[batch_ids[self.batch_size:]]
y = y1 * y_l + y2 *(1 - y_l)
 return X, y
def get_sample(self):
return self.__data_generation(self.__get_exploration_order() [i * self.batch_size * 2:(i + 1)* self.batch_size * 2] )
train['revenue'] = train['item_price'] * train['item_cnt_day']<merge>
BATCH_SIZE = 32
def create_datagen() :
return ImageDataGenerator(
 preprocessing_function=seq.augment_image 
)
 if TRAINING:
data_generator = create_datagen().flow(x_train, y_train, batch_size=BATCH_SIZE)
 mixup_generator = MixupGenerator(x_train, y_train, seq=seq, batch_size=BATCH_SIZE, alpha=0.2 )()
ts = time.time()
group = train.groupby(['date_block_num','shop_id','item_id'] ).agg({'item_cnt_day': ['sum']})
 group.columns = ['item_cnt_month']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=cols, how='left')
 matrix['item_cnt_month'] =(matrix['item_cnt_month'] 
.fillna(0)
.clip(0,20)
.astype(np.float16))
time.time() - ts<data_type_conversions>
true_labels = np.array([1, 0, 1, 1, 0, 1])
 pred_labels = np.array([1, 0, 0, 0, 0, 1] )
test['date_block_num'] = 34
test['date_block_num'] = test['date_block_num'].astype(np.int8)
 test['shop_id'] = test['shop_id'].astype(np.int8)
 test['item_id'] = test['item_id'].astype(np.int16 )<concatenate>
accuracy_score(true_labels, pred_labels )
ts = time.time()
matrix = pd.concat([matrix, test], ignore_index=True, sort=False, keys=cols)
 matrix.fillna(0, inplace=True)
 time.time() - ts<data_type_conversions>
cohen_kappa_score(true_labels, pred_labels )
ts = time.time()
matrix = pd.merge(matrix, shops, on=['shop_id'], how='left')
 matrix = pd.merge(matrix, items, on=['item_id'], how='left')
 matrix = pd.merge(matrix, cats, on=['item_category_id'], how='left')
 matrix['city_code'] = matrix['city_code'].astype(np.int8)
 matrix['item_category_id'] = matrix['item_category_id'].astype(np.int8)
 matrix['type_code'] = matrix['type_code'].astype(np.int8)
 matrix['subtype_code'] = matrix['subtype_code'].astype(np.int8)
 time.time() - ts<merge>
class Metrics(Callback):
def on_train_begin(self, logs={}):
self.val_kappas = []
def on_epoch_end(self, epoch, logs={}):
X_val, y_val = self.validation_data[:2]
y_val =(y_val.sum(axis=1)- 1 ).clip(0, 4)
 y_pred = self.model.predict(X_val)> 0.5
y_pred =(y_pred.astype(int ).sum(axis=1)- 1 ).clip(0, 4)
 _val_kappa = cohen_kappa_score(
 y_val,
y_pred,
weights='quadratic' 
)
 self.val_kappas.append(_val_kappa)
 print(f"val_kappa: {_val_kappa:.4f}")
 if _val_kappa == max(self.val_kappas):
print("Validation Kappa has improved.Saving model.")
 self.model.save('model.h5')
 return
def lag_feature(df, lags, col):
tmp = df[['date_block_num','shop_id','item_id',col]]
for i in lags:
shifted = tmp.copy()
shifted.columns = ['date_block_num','shop_id','item_id', col+'_lag_'+str(i)]
shifted['date_block_num'] += i
df = pd.merge(df, shifted, on=['date_block_num','shop_id','item_id'], how='left')
 return df<merge>
base_model = Xception(
 weights=None,
include_top=False,
input_shape=(224,224,3)
)
 base_model.load_weights(".. /input/keras-pretrained-models/xception_weights_tf_dim_ordering_tf_kernels_notop.h5" )
ts = time.time()
group = matrix.groupby(['date_block_num'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num'], how='left')
 matrix['date_avg_item_cnt'] = matrix['date_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_avg_item_cnt')
 matrix.drop(['date_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
def build_model() :
model = Sequential()
model.add(base_model)
 model.add(layers.GlobalAveragePooling2D())
 model.add(layers.Dropout(0.5))
model.add(layers.Dense(5, activation='sigmoid'))
model.compile(
 loss='binary_crossentropy',
optimizer=Adam(lr=0.00005),
metrics=['accuracy'] 
)
 return model
ts = time.time()
group = matrix.groupby(['date_block_num', 'item_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_item_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','item_id'], how='left')
 matrix['date_item_avg_item_cnt'] = matrix['date_item_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1,2,3,6,12], 'date_item_avg_item_cnt')
 matrix.drop(['date_item_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
if TRAINING:
kappa_metrics = Metrics()
rlr = callbacks.ReduceLROnPlateau(factor=0.5, patience=4, verbose=1)
 es = callbacks.EarlyStopping(patience=10, verbose=1, mode="min")
 history = model.fit_generator(
 data_generator,
steps_per_epoch=x_train.shape[0] / BATCH_SIZE,
epochs=200,
validation_data=(x_val, y_val),
callbacks=[kappa_metrics, rlr, es]
)
ts = time.time()
group = matrix.groupby(['date_block_num', 'shop_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_shop_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','shop_id'], how='left')
 matrix['date_shop_avg_item_cnt'] = matrix['date_shop_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1,2,3,6,12], 'date_shop_avg_item_cnt')
 matrix.drop(['date_shop_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
if TRAINING:
with open('history.json', 'w')as f:
json.dump(str(history.history), f)
 history_df = pd.DataFrame(history.history)
 history_df[['loss', 'val_loss']].plot()
history_df[['acc', 'val_acc']].plot()
ts = time.time()
group = matrix.groupby(['date_block_num', 'item_category_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_cat_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','item_category_id'], how='left')
 matrix['date_cat_avg_item_cnt'] = matrix['date_cat_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_cat_avg_item_cnt')
 matrix.drop(['date_cat_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
<merge><EOS> 
if TRAINING:
weights_path = 'model.h5'
else:
weights_path = '.. /input/weights/model.h5'
model.load_weights(weights_path)
 y_test = model.predict(x_test)> 0.5
y_test =(y_test.astype(int ).sum(axis=1)- 1 ).clip(0, 4)
 test_df['diagnosis'] = y_test
test_df.to_csv('submission.csv',index=False )