Kaggle competition: APTOS 2019 Blindness Detection
<SOS> metric: QuadraticWeightedKappa Kaggle data source: aptos-2019-blindness-detection<compute_test_metric>
def seed_everything(seed=0):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 set_random_seed(0)
 seed = 0
seed_everything(seed)
 %matplotlib inline
sns.set(style="whitegrid")
 warnings.filterwarnings("ignore")
 sys.path.append(os.path.abspath('.. /input/efficientnet/efficientnet-master/efficientnet-master/'))
rmse_train = np.sqrt(mean_squared_error(y_train_level2, train_preds))
print('RMSE Train: %f' % rmse_train)
<train_model>
hold_out_set = pd.read_csv('.. /input/aptos-data-split/hold-out.csv')
 X_train = hold_out_set[hold_out_set['set'] == 'train']
X_val = hold_out_set[hold_out_set['set'] == 'validation']
test = pd.read_csv('.. /input/aptos2019-blindness-detection/test.csv')
 print('Number of train samples: ', X_train.shape[0])
 print('Number of validation samples: ', X_val.shape[0])
 print('Number of test samples: ', test.shape[0])
 X_train["id_code"] = X_train["id_code"].apply(lambda x: x + ".png")
 X_val["id_code"] = X_val["id_code"].apply(lambda x: x + ".png")
 test["id_code"] = test["id_code"].apply(lambda x: x + ".png")
 display(X_train.head() )
print(' mean train preds : %f ' ,(train_preds.mean()))<compute_test_metric>
HEIGHT = 224
WIDTH = 224
CHANNELS = 3
TTA_STEPS = 10
weights_path_list = ['.. /input/aptos-5fold-224-oldnew/effNetB5_img224_fold1.h5',
'.. /input/aptos-5fold-224-oldnew/effNetB5_img224_fold2.h5',
'.. /input/aptos-5fold-224-oldnew/effNetB5_img224_fold3.h5',
'.. /input/aptos-5fold-224-oldnew/effNetB5_img224_fold4.h5',
'.. /input/aptos-5fold-224-oldnew/effNetB5_img224_fold5.h5']
print(' mean test preds : %f ' ,(test_preds.mean()))<install_modules>
labels = ['0 - No DR', '1 - Mild', '2 - Moderate', '3 - Severe', '4 - Proliferative DR']
def plot_confusion_matrix(train, validation, labels=labels):
train_labels, train_preds = train
validation_labels, validation_preds = validation
fig,(ax1, ax2)= plt.subplots(1, 2, sharex='col', figsize=(24, 7))
train_cnf_matrix = confusion_matrix(train_labels, train_preds)
 validation_cnf_matrix = confusion_matrix(validation_labels, validation_preds)
 train_cnf_matrix_norm = train_cnf_matrix.astype('float')/ train_cnf_matrix.sum(axis=1)[:, np.newaxis]
validation_cnf_matrix_norm = validation_cnf_matrix.astype('float')/ validation_cnf_matrix.sum(axis=1)[:, np.newaxis]
train_df_cm = pd.DataFrame(train_cnf_matrix_norm, index=labels, columns=labels)
 validation_df_cm = pd.DataFrame(validation_cnf_matrix_norm, index=labels, columns=labels)
 sns.heatmap(train_df_cm, annot=True, fmt='.2f', cmap="Blues",ax=ax1 ).set_title('Train')
 sns.heatmap(validation_df_cm, annot=True, fmt='.2f', cmap=sns.cubehelix_palette(8),ax=ax2 ).set_title('Validation')
 plt.show()
def evaluate_model(train, validation):
train_labels, train_preds = train
validation_labels, validation_preds = validation
print("Train Cohen Kappa score: %.3f" % cohen_kappa_score(train_preds, train_labels, weights='quadratic'))
print("Validation Cohen Kappa score: %.3f" % cohen_kappa_score(validation_preds, validation_labels, weights='quadratic'))
print("Complete set Cohen Kappa score: %.3f" % cohen_kappa_score(np.append(train_preds, validation_preds), np.append(train_labels, validation_labels), weights='quadratic'))
def classify(x):
if x < 0.5:
return 0
elif x < 1.5:
return 1
elif x < 2.5:
return 2
elif x < 3.5:
return 3
return 4
def ensemble_preds(model_list, generator):
preds_ensemble = []
for model in model_list:
generator.reset()
preds = model.predict_generator(generator, steps=generator.n)
 preds_ensemble.append(preds)
 return np.mean(preds_ensemble, axis=0)
 def apply_tta(model, generator, steps=5):
step_size = generator.n//generator.batch_size
preds_tta = []
for i in range(steps):
generator.reset()
preds = model.predict_generator(generator, steps=step_size)
 preds_tta.append(preds)
 return np.mean(preds_tta, axis=0)
 def test_ensemble_preds(model_list, generator, steps=5):
preds_ensemble = []
for model in model_list:
preds = apply_tta(model, generator, steps)
 preds_ensemble.append(preds)
 return np.mean(preds_ensemble, axis=0 )
!pip install googletrans
!pip install pytrends<set_options>
datagen=ImageDataGenerator(rescale=1./255,
rotation_range=360,
horizontal_flip=True,
vertical_flip=True)
 train_generator=datagen.flow_from_dataframe(
 dataframe=X_train,
directory=train_dest_path,
x_col="id_code",
y_col="diagnosis",
class_mode="raw",
batch_size=1,
shuffle=False,
target_size=(HEIGHT, WIDTH),
seed=seed)
 valid_generator=datagen.flow_from_dataframe(
 dataframe=X_val,
directory=validation_dest_path,
x_col="id_code",
y_col="diagnosis",
class_mode="raw",
batch_size=1,
shuffle=False,
target_size=(HEIGHT, WIDTH),
seed=seed)
 test_generator=datagen.flow_from_dataframe(
 dataframe=test,
directory=test_dest_path,
x_col="id_code",
batch_size=1,
class_mode=None,
shuffle=False,
target_size=(HEIGHT, WIDTH),
seed=seed )
%matplotlib inline 
<define_variables>
def create_model(input_shape, weights_path):
input_tensor = Input(shape=input_shape)
 base_model = EfficientNetB5(weights=None,
include_top=False,
input_tensor=input_tensor)
 x = GlobalAveragePooling2D()(base_model.output)
 final_output = Dense(1, activation='linear', name='final_output' )(x)
 model = Model(input_tensor, final_output)
 model.load_weights(weights_path)
 return model
pd.set_option('display.max_columns', 100)
 pd.set_option('display.max_rows', 100)
 ori_data_path = '/kaggle/input/competitive-data-science-predict-future-sales/'
model_path = '/'
processed_data_path = '/'
tables_path = '/'
items_IP_path = '.. /input/predict-future-sales-ip-groups/items_IPs_v1.csv'
items_IP_trend_path = '.. /input/predict-future-sales-ip-groups/items_IP_trend.csv'<load_from_csv>
model_list = []
for weights_path in weights_path_list:
model_list.append(create_model(input_shape=(HEIGHT, WIDTH, CHANNELS), weights_path=weights_path))
df = pd.read_csv(ori_data_path+'sales_train.csv')
 df_test = pd.read_csv(ori_data_path+'test.csv')
 sample_submission = pd.read_csv(ori_data_path+'sample_submission.csv')
 item_categories = pd.read_csv(ori_data_path+'item_categories.csv')
 items = pd.read_csv(ori_data_path+'items.csv')
 shops = pd.read_csv(ori_data_path+'shops.csv' )<data_type_conversions>
preds_ensemble = ensemble_preds(model_list, train_generator)
 preds_ensemble = [classify(x)for x in preds_ensemble]
train_preds = pd.DataFrame({'label':train_generator.labels, 'pred':preds_ensemble})
 preds_ensemble = ensemble_preds(model_list, valid_generator)
 preds_ensemble = [classify(x)for x in preds_ensemble]
validation_preds = pd.DataFrame({'label':valid_generator.labels, 'pred':preds_ensemble} )
df[['date_block_num','shop_id']] = df[['date_block_num','shop_id']].astype('int8')
 df[['item_cnt_day','item_id']] = df[['item_cnt_day','item_id']].astype('int16')
 df['item_price'] = df['item_price'].astype('float32' )<count_duplicates>
evaluate_model(( train_preds['label'], train_preds['pred']),(validation_preds['label'], validation_preds['pred']))
shop_test = df_test['shop_id'].nunique()
item_test = df_test['item_id'].nunique()
total_product_test = df_test[['shop_id','item_id']].drop_duplicates()
shop_train = df['shop_id'].nunique()
item_train = df['item_id'].nunique()
total_product_train = df[['shop_id','item_id']].drop_duplicates()
print('We need to predict products from {} shops and {} kinds of products(total shop-item {} pairs)'.format(shop_test,item_test,total_product_test.shape[0]))
print('given that historical daily sale number from {} shops and {} kinds of products(total shop-item {} pairs)'.format(shop_train,item_train,total_product_train.shape[0]))<merge>
preds = test_ensemble_preds(model_list, test_generator, TTA_STEPS)
 predictions = [classify(x)for x in preds]
results = pd.DataFrame({'id_code':test['id_code'], 'diagnosis':predictions})
 results['id_code'] = results['id_code'].map(lambda x: str(x)[:-4] )
common_pairs = total_product_test.merge(total_product_train,how='inner',on=['shop_id','item_id'] ).shape[0]
test_only_pairs = total_product_test.shape[0] - common_pairs
print(f'{common_pairs} pairs of products are shared by historical and to be predicted.')
 print(f'{test_only_pairs} pairs are new pairs, so there is no corresponding historical data for prediciton.' )<merge>
if os.path.exists(train_dest_path):
shutil.rmtree(train_dest_path)
 if os.path.exists(validation_dest_path):
shutil.rmtree(validation_dest_path)
 if os.path.exists(test_dest_path):
shutil.rmtree(test_dest_path )
<drop_column><EOS> 
results.to_csv('submission.csv', index=False)
 display(results.head() )