Kaggle competition: APTOS 2019 Blindness Detection
<SOS> metric: QuadraticWeightedKappa Kaggle data source: aptos-2019-blindness-detection<load_from_csv>
import os
from keras.applications.xception import Xception
from keras.models import Model
from keras.layers import Input, Activation, Dropout, Flatten, Dense, GlobalAveragePooling2D
from keras import optimizers
from keras.callbacks import ModelCheckpoint
from keras.utils import np_utils
from keras.models import load_model
import cv2
from keras import backend as K
import matplotlib.pyplot as plt
from matplotlib.colors import rgb_to_hsv, hsv_to_rgb
import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split
import gc
nrows = None
items = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/items.csv')
 shops = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/shops.csv')
 cats = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/item_categories.csv')
 try:
train = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/sales_train_v2.csv', nrows=nrows)
 except:
train = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/sales_train.csv', nrows=nrows)
 test = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/test.csv' ).set_index('ID' )<define_variables>
img = np.loadtxt(".. /input/aptos2019-blindness-detection/train.csv",
delimiter=",",
skiprows=1,
usecols=(0),
dtype = "str" 
)
 img
print(f"{len(set(train.shop_id)- set(test.shop_id)) } shops only in train.")
 print(f"{len(train.item_cnt_day[train.item_cnt_day < 0])} negative items_cnt_day")
 print(f"We need to predict clipped into [0, 20] range item_cnt_month column" )<filter>
label = np.loadtxt(".. /input/aptos2019-blindness-detection/train.csv",
delimiter=",",
skiprows=1,
usecols=(1),
dtype = "int" 
)
 label
train = train[train.item_price<48000]
train = train[train.item_cnt_day<1000]<categorify>
img_label_trains = []
img_label_validations = []
for i in range(4):
data_train, data_test, labels_train, labels_test = train_test_split(img, label, train_size=0.85,random_state=i*5,stratify=label)
 img_label_train = np.stack([data_train, labels_train],axis=1)
 img_label_validation = np.stack([data_test, labels_test],axis=1)
 img_label_trains.append(img_label_train)
 img_label_validations.append(img_label_validation )
cats.loc[32, "item_category_name"] = 'Карты оплаты -(Кино, Музыка, Игры)'
cats['split'] = cats['item_category_name'].str.split('-')
 cats['type'] = cats['split'].map(lambda x: x[0].strip())
 cats['type_code'] = LabelEncoder().fit_transform(cats['type'])
 cats['subtype'] = cats['split'].map(lambda x: x[1].strip() if len(x)> 1 else x[0].strip())
 cats['subtype_code'] = LabelEncoder().fit_transform(cats['subtype'])
 cats = cats[['item_category_id','type_code', 'subtype_code']]
cats<categorify>
print(np.count_nonzero(img_label_trains[0][:,1] == "0"))
print(np.count_nonzero(img_label_trains[0][:,1] == "1"))
print(np.count_nonzero(img_label_trains[0][:,1] == "2"))
print(np.count_nonzero(img_label_trains[0][:,1] == "3"))
print(np.count_nonzero(img_label_trains[0][:,1] == "4"))
print(np.count_nonzero(img_label_validations[0][:,1] == "0"))
print(np.count_nonzero(img_label_validations[0][:,1] == "1"))
print(np.count_nonzero(img_label_validations[0][:,1] == "2"))
print(np.count_nonzero(img_label_validations[0][:,1] == "3"))
print(np.count_nonzero(img_label_validations[0][:,1] == "4"))
shops.loc[shops.shop_name == 'Сергиев Посад ТЦ "7Я"', 'shop_name'] = 'СергиевПосад ТЦ "7Я"'
shops['city'] = shops['shop_name'].str.split(' ' ).map(lambda x: x[0])
 shops.loc[shops.city == '!Якутск', 'city'] = 'Якутск'
shops['city_code'] = LabelEncoder().fit_transform(shops['city'])
 shops = shops[['shop_id','city_code']]<feature_engineering>
def data_generator(image_lines, batch_size, abs_path, img_width, img_height, data_aug):

 n = len(image_lines)
 i = 0
while True:
image_data = []
label_data = []
for b in range(batch_size):
if i==0:
np.random.shuffle(image_lines)
 image, label = get_random_data(image_lines[i], abs_path, img_width, img_height, data_aug)
 image_data.append(image)
 label_data.append(label)
 i =(i+1)% n
image_data = np.array(image_data)
 label_data = np.array(label_data)
 yield image_data, label_data
def data_generator_wrapper(image_lines, batch_size, abs_path, img_width, img_height, data_aug):
n = len(image_lines)
 if n==0 or batch_size<=0: return None
return data_generator(image_lines, batch_size, abs_path, img_width, img_height, data_aug )
train.loc[train.shop_id == 0, 'shop_id'] = 57
test.loc[test.shop_id == 0, 'shop_id'] = 57
train.loc[train.shop_id == 1, 'shop_id'] = 58
test.loc[test.shop_id == 1, 'shop_id'] = 58
train.loc[train.shop_id == 10, 'shop_id'] = 11
test.loc[test.shop_id == 10, 'shop_id'] = 11<data_type_conversions>
img_width, img_height = 449, 449
num_train = len(img_label_trains[0])
 num_val = len(img_label_validations[0])
 batch_size = 4
print(num_train, num_val)
 abs_path = ".. /input/aptos2019-blindness-detection/train_images/"
matrix = []
cols = ['date_block_num','shop_id','item_id']
for i in range(34):
sales = train[train.date_block_num==i]
matrix.append(np.array(list(product([i],
sales.shop_id.unique() ,
sales.item_id.unique())) ,
dtype='int16'))
matrix = pd.DataFrame(np.vstack(matrix), columns=cols)
 matrix['date_block_num'] = matrix['date_block_num'].astype(np.int8)
 matrix['shop_id'] = matrix['shop_id'].astype(np.int8)
 matrix['item_id'] = matrix['item_id'].astype(np.int16)
 matrix.sort_values(cols,inplace=True )<merge>
models = []
for i in range(4):
input_tensor = Input(shape=(img_height, img_width, 3))
xception_model = Xception(include_top=False, weights=None, input_tensor=input_tensor)
 xception_model.load_weights(".. /input/keras-pretrained-models/xception_weights_tf_dim_ordering_tf_kernels_notop.h5")
 x = xception_model.output
x = GlobalAveragePooling2D()(x)
 x = Dense(1024, activation='relu' )(x)
 x = Dropout(0.3 )(x)
 outputs = Dense(5, activation='softmax' )(x)
 model = Model(inputs=xception_model.input, outputs=outputs)
 model.compile(optimizer=optimizers.SGD(lr=0.001,momentum=0.9),
loss='categorical_crossentropy',
metrics=['accuracy'])
 model.summary()
models.append(model )
group = train.groupby(['date_block_num','shop_id','item_id'] ).agg({'item_cnt_day': ['sum']})
 group.columns = ['item_cnt_month']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=cols, how='left')
 matrix['item_cnt_month'] =(matrix['item_cnt_month'] 
.fillna(0)
.clip(0,20)
.astype(np.float16))<data_type_conversions>
models[0].fit_generator(data_generator_wrapper(img_label_trains[0], batch_size, abs_path, img_width, img_height, True),
steps_per_epoch=max(1, num_train//batch_size),
validation_data=data_generator_wrapper(img_label_validations[0], batch_size, abs_path, img_width, img_height, True),
validation_steps=max(1, num_val//batch_size),
epochs=5,
initial_epoch=0,
class_weight=[1,4.8,1.8,9.1,6.3] )
test['date_block_num'] = 34
test['date_block_num'] = test['date_block_num'].astype(np.int8)
 test['shop_id'] = test['shop_id'].astype(np.int8)
 test['item_id'] = test['item_id'].astype(np.int16)
 matrix = pd.concat([matrix, test], ignore_index=True, sort=False, keys=cols)
 matrix.fillna(0, inplace=True )<data_type_conversions>
models[0].fit_generator(data_generator_wrapper(img_label_trains[0], batch_size, abs_path, img_width, img_height, True),
steps_per_epoch=max(1, num_train//batch_size),
validation_data=data_generator_wrapper(img_label_validations[0], batch_size, abs_path, img_width, img_height, True),
validation_steps=max(1, num_val//batch_size),
epochs=5,
initial_epoch=0,
class_weight=[1,4.8,1.8,9.1,6.3] )
matrix = pd.merge(matrix, shops, on=['shop_id'], how='left')
 matrix = pd.merge(matrix, items, on=['item_id'], how='left')
 matrix = pd.merge(matrix, cats, on=['item_category_id'], how='left')
 matrix['city_code'] = matrix['city_code'].astype(np.int8)
 matrix['item_category_id'] = matrix['item_category_id'].astype(np.int8)
 matrix['type_code'] = matrix['type_code'].astype(np.int8)
 matrix['subtype_code'] = matrix['subtype_code'].astype(np.int8)
 matrix = matrix.drop(['item_name'], axis=1 )<merge>
models[0].fit_generator(data_generator_wrapper(img_label_trains[0], batch_size, abs_path, img_width, img_height, True),
steps_per_epoch=max(1, num_train//batch_size),
validation_data=data_generator_wrapper(img_label_validations[0], batch_size, abs_path, img_width, img_height, True),
validation_steps=max(1, num_val//batch_size),
epochs=5,
initial_epoch=0,
class_weight=[1,4.8,1.8,9.1,6.3] )
lookback_range = list(range(1, 7 + 1))
new_features = []
for diff in tqdm(lookback_range):
feature_name = 'prev_shopitem_sales_' + str(diff)
 mx2 = matrix.copy()
mx2.loc[:, 'date_block_num'] += diff
mx2.rename(columns={'item_cnt_month': feature_name}, inplace=True)
 matrix = matrix.merge(mx2[['shop_id', 'item_id', 'date_block_num', feature_name]], on = ['shop_id', 'item_id', 'date_block_num'], how = 'left')
 matrix[feature_name] = matrix[feature_name].fillna(0)
 new_features.append(feature_name)
 groups = matrix.groupby(by = ['item_id', 'date_block_num'])
 for diff in tqdm(lookback_range):
feature_name = 'prev_item_sales_' + str(diff)
 result = groups.agg({'item_cnt_month':'mean'})
 result = result.reset_index()
result.loc[:, 'date_block_num'] += diff
result.rename(columns={'item_cnt_month': feature_name}, inplace=True)
 matrix = matrix.merge(result, on = ['item_id', 'date_block_num'], how = 'left')
 matrix[feature_name] = matrix[feature_name].fillna(0)
 new_features.append(feature_name)
 groups = matrix.groupby(by = ['item_id', 'date_block_num'])
 for diff in tqdm(lookback_range):
feature_name = '_prev_item_sales_' + str(diff)
 result = groups.agg({'item_cnt_month':'mean'})
 result = result.reset_index()
result.loc[:, 'date_block_num'] += diff
result.rename(columns={'item_cnt_month': feature_name}, inplace=True)
 matrix = matrix.merge(result, on = ['item_id', 'date_block_num'], how = 'left')
 matrix[feature_name] = matrix[feature_name].fillna(0)
 new_features.append(feature_name )<feature_engineering>
models[0].fit_generator(data_generator_wrapper(img_label_trains[0], batch_size, abs_path, img_width, img_height, True),
steps_per_epoch=max(1, num_train//batch_size),
validation_data=data_generator_wrapper(img_label_validations[0], batch_size, abs_path, img_width, img_height, True),
validation_steps=max(1, num_val//batch_size),
epochs=5,
initial_epoch=0,
class_weight=[1,4.8,1.8,9.1,6.3] )
me_cols = ['shop_id', 'item_id', 'city_code', 'item_category_id', 'type_code', 'subtype_code']
for cl in me_cols:
me_col_name = "me_" + cl
matrix.loc[:, me_col_name] = matrix[cl].map(matrix[cl].value_counts().apply(lambda x: x/len(matrix[cl])) )<drop_column>
models[1].fit_generator(data_generator_wrapper(img_label_trains[1], batch_size, abs_path, img_width, img_height, True),
steps_per_epoch=max(1, num_train//batch_size),
validation_data=data_generator_wrapper(img_label_validations[1], batch_size, abs_path, img_width, img_height, True),
validation_steps=max(1, num_val//batch_size),
epochs=5,
initial_epoch=0,
class_weight=[1,4.8,1.8,9.1,6.3] )
del items, shops, cats, train
matrix = reduce_mem_usage(matrix)
 test = reduce_mem_usage(test )<prepare_x_and_y>
models[1].fit_generator(data_generator_wrapper(img_label_trains[1], batch_size, abs_path, img_width, img_height, True),
steps_per_epoch=max(1, num_train//batch_size),
validation_data=data_generator_wrapper(img_label_validations[1], batch_size, abs_path, img_width, img_height, True),
validation_steps=max(1, num_val//batch_size),
epochs=5,
initial_epoch=0,
class_weight=[1,4.8,1.8,9.1,6.3] )
X_train = matrix[matrix.date_block_num <= 33].drop(['item_cnt_month'], axis=1)
 y = matrix[matrix.date_block_num <= 33]['item_cnt_month']
X_test = matrix[matrix.date_block_num == 34].drop(['item_cnt_month'], axis=1)
 del matrix
gc.collect()<normalization>
models[1].fit_generator(data_generator_wrapper(img_label_trains[1], batch_size, abs_path, img_width, img_height, True),
steps_per_epoch=max(1, num_train//batch_size),
validation_data=data_generator_wrapper(img_label_validations[1], batch_size, abs_path, img_width, img_height, True),
validation_steps=max(1, num_val//batch_size),
epochs=5,
initial_epoch=0,
class_weight=[1,4.8,1.8,9.1,6.3] )
x_stsc = StandardScaler().fit(pd.concat([X_train, X_test]))
X_train = x_stsc.transform(X_train)
 X_test = x_stsc.transform(X_test )<choose_model_class>
models[1].fit_generator(data_generator_wrapper(img_label_trains[1], batch_size, abs_path, img_width, img_height, True),
steps_per_epoch=max(1, num_train//batch_size),
validation_data=data_generator_wrapper(img_label_validations[1], batch_size, abs_path, img_width, img_height, True),
validation_steps=max(1, num_val//batch_size),
epochs=5,
initial_epoch=0,
class_weight=[1,4.8,1.8,9.1,6.3] )
cv = KFold(3, shuffle=True )<train_model>
models[2].fit_generator(data_generator_wrapper(img_label_trains[2], batch_size, abs_path, img_width, img_height, True),
steps_per_epoch=max(1, num_train//batch_size),
validation_data=data_generator_wrapper(img_label_validations[2], batch_size, abs_path, img_width, img_height, True),
validation_steps=max(1, num_val//batch_size),
epochs=5,
initial_epoch=0,
class_weight=[1,4.8,1.8,9.1,6.3] )
def linear_model(
 X,
X_test,
y,
model,
params,
folds,
columns=None):
for fold_n,(train_index, valid_index)in enumerate(folds.split(X)) :
print(f'Fold {fold_n + 1} started at {time.ctime() }')
 result_dict = {}
oof = np.zeros(len(X))
prediction = np.zeros(len(X_test))
scores = []
feature_importance = pd.DataFrame()
X_train, X_valid = X[train_index], X[valid_index]
y_train, y_valid = y[train_index], y[valid_index]
model.fit(X_train, y_train)
 y_pred_valid = model.predict(X_valid)
 y_pred = model.predict(X_test)
 oof[valid_index] = y_pred_valid.reshape(-1,)
 scores.append(rmse(y_valid, y_pred_valid))
prediction += y_pred
gc.collect()
prediction /= folds.n_splits
print('CV mean score: {0:.4f}, std: {1:.4f}.'.format(np.mean(scores), np.std(scores)))
 result_dict['oof'] = oof
result_dict['prediction'] = prediction
result_dict['scores'] = scores
return result_dict<choose_model_class>
models[2].fit_generator(data_generator_wrapper(img_label_trains[2], batch_size, abs_path, img_width, img_height, True),
steps_per_epoch=max(1, num_train//batch_size),
validation_data=data_generator_wrapper(img_label_validations[2], batch_size, abs_path, img_width, img_height, True),
validation_steps=max(1, num_val//batch_size),
epochs=5,
initial_epoch=0,
class_weight=[1,4.8,1.8,9.1,6.3] )
linreg_best_params= {
'n_jobs': -1
}
ridge_best_params = {
'alpha': 0.7,
'solver': 'lsqr',
'random_state': RANDOM_SEED
}
lr_model = LinearRegression(**linreg_best_params)
 ridge_model = Ridge(**ridge_best_params)
 linreg_results = linear_model(X_train, X_test, y,
lr_model,
linreg_best_params,
cv)
 ridge_results = linear_model(X_train, X_test, y,
ridge_model,
linreg_best_params,
cv)
<set_options>
models[2].fit_generator(data_generator_wrapper(img_label_trains[2], batch_size, abs_path, img_width, img_height, True),
steps_per_epoch=max(1, num_train//batch_size),
validation_data=data_generator_wrapper(img_label_validations[2], batch_size, abs_path, img_width, img_height, True),
validation_steps=max(1, num_val//batch_size),
epochs=5,
initial_epoch=0,
class_weight=[1,4.8,1.8,9.1,6.3] )
gc.collect()<split>
models[2].fit_generator(data_generator_wrapper(img_label_trains[2], batch_size, abs_path, img_width, img_height, True),
steps_per_epoch=max(1, num_train//batch_size),
validation_data=data_generator_wrapper(img_label_validations[2], batch_size, abs_path, img_width, img_height, True),
validation_steps=max(1, num_val//batch_size),
epochs=5,
initial_epoch=0,
class_weight=[1,4.8,1.8,9.1,6.3] )
def random_forest_regressor(
 X,
X_test,
y,
params,
folds,
columns=None):
for fold_n,(train_index, valid_index)in enumerate(folds.split(X)) :
print(f'Fold {fold_n + 1} started at {time.ctime() }')
 result_dict = {}
oof = np.zeros(len(X))
prediction = np.zeros(len(X_test))
scores = []
feature_importance = pd.DataFrame()
if type(X)== np.ndarray:
X_train, X_valid = X[train_index], X[valid_index]
y_train, y_valid = y[train_index], y[valid_index]
else:
X_train, X_valid = X[columns].iloc[train_index], X[columns].iloc[valid_index]
y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]
model = RandomForestRegressor(**params)
 model.fit(X_train, y_train)
 y_pred_valid = model.predict(X_valid)
 y_pred = model.predict(X_test)
 oof[valid_index] = y_pred_valid.reshape(-1,)
 scores.append(rmse(y_valid, y_pred_valid))
prediction += y_pred
gc.collect()
prediction /= folds.n_splits
print('CV mean score: {0:.4f}, std: {1:.4f}.'.format(np.mean(scores), np.std(scores)))
 result_dict['oof'] = oof
result_dict['prediction'] = prediction
result_dict['scores'] = scores
gc.collect()
return result_dict 
<train_on_grid>
models[3].fit_generator(data_generator_wrapper(img_label_trains[2], batch_size, abs_path, img_width, img_height, True),
steps_per_epoch=max(1, num_train//batch_size),
validation_data=data_generator_wrapper(img_label_validations[2], batch_size, abs_path, img_width, img_height, True),
validation_steps=max(1, num_val//batch_size),
epochs=5,
initial_epoch=0,
class_weight=[1,4.8,1.8,9.1,6.3] )
rf_best_params= {
'bootstrap': True,
'max_depth': 40,
'max_features': 'auto',
'min_samples_leaf': 4,
'min_samples_split': 10,
'n_estimators': 4,
'n_jobs': -1,
'verbose': 2
}
rf_results = random_forest_regressor(X_train, X_test, y,
rf_best_params,
cv )<train_model>
models[3].fit_generator(data_generator_wrapper(img_label_trains[2], batch_size, abs_path, img_width, img_height, True),
steps_per_epoch=max(1, num_train//batch_size),
validation_data=data_generator_wrapper(img_label_validations[2], batch_size, abs_path, img_width, img_height, True),
validation_steps=max(1, num_val//batch_size),
epochs=5,
initial_epoch=0,
class_weight=[1,4.8,1.8,9.1,6.3] )
params= {
'num_leaves': 64,
"max_depth": 8,
'max_bin': 32,
"iterations": 200,
"data_random_seed":0,
'n_estimators': 1000
}
lgb_results = trainer(X_train, X_test, y, params, cv )<prepare_x_and_y>
models[3].fit_generator(data_generator_wrapper(img_label_trains[2], batch_size, abs_path, img_width, img_height, True),
steps_per_epoch=max(1, num_train//batch_size),
validation_data=data_generator_wrapper(img_label_validations[2], batch_size, abs_path, img_width, img_height, True),
validation_steps=max(1, num_val//batch_size),
epochs=5,
initial_epoch=0,
class_weight=[1,4.8,1.8,9.1,6.3] )
oof_preds = np.array([ridge_results['oof'],
linreg_results['oof'],
rf_results['oof'],
lgb_results['oof']
] ).T
preds = np.array([ridge_results['prediction'],
linreg_results['prediction'],
rf_results['prediction'],
lgb_results['prediction']
] ).T<choose_model_class>
models[3].fit_generator(data_generator_wrapper(img_label_trains[2], batch_size, abs_path, img_width, img_height, True),
steps_per_epoch=max(1, num_train//batch_size),
validation_data=data_generator_wrapper(img_label_validations[2], batch_size, abs_path, img_width, img_height, True),
validation_steps=max(1, num_val//batch_size),
epochs=5,
initial_epoch=0,
class_weight=[1,4.8,1.8,9.1,6.3] )
lvl2_model = LinearRegression()
lvl2_model.fit(oof_preds, y)
 lvl2_model.score(oof_preds, y)
 lvl2_result = lvl2_model.predict(preds )<save_to_csv>
img_test = np.loadtxt(".. /input/aptos2019-blindness-detection/test.csv",
delimiter=",",
skiprows=1,
dtype = "str"
)
submission = pd.DataFrame({
"ID": test.index,
"item_cnt_month": lvl2_result
})
 submission['item_cnt_month'] = submission['item_cnt_month'].clip(0, 20)
 submission.to_csv('submission.csv',index=False )<define_variables>
test_abs_path = ".. /input/aptos2019-blindness-detection/test_images/"
data = []
for i in range(len(img_test)) :
image_file = test_abs_path + img_test[i] + ".png"
seed_image = cv2.imread(image_file)
 seed_image = cv2.cvtColor(seed_image, cv2.COLOR_BGR2RGB)
 seed_image = cv2.resize(seed_image, dsize=(img_width, img_height))
seed_image = np.expand_dims(seed_image, axis=0)
 seed_image = seed_image / 255
predict1 = models[0].predict(seed_image)
 predict2 = models[1].predict(seed_image)
 predict3 = models[2].predict(seed_image)
 predict4 = models[3].predict(seed_image)
 predict_mean =(predict1+predict2+predict3+predict4)/4
x = np.array([img_test[i], np.argmax(predict_mean)])
 data.append(x)
 data = np.array(data )
<import_modules>
columns = ['id_code', 'diagnosis']
name = 'sample'
d = pd.DataFrame(data=data, columns=columns, dtype='str' )
tqdm.pandas(desc='Progress')
<init_hyperparams>
d.to_csv("submission.csv",index=False )
embed_size = 300
max_features = 120000
maxlen = 70
batch_size = 512
n_epochs = 5
n_splits = 5
SEED =1029
debug = 0
warnings.filterwarnings('ignore')
 M_Last = True
print(( n_splits - 1)/n_splits * n_epochs )<choose_model_class>
df = pd.read_csv("submission.csv")
 print(df )