Kaggle competition: APTOS 2019 Blindness Detection
<SOS> metric: QuadraticWeightedKappa Kaggle data source: aptos-2019-blindness-detection<predict_on_test>
DATA_PATH = ".. /input/aptos2019-blindness-detection/"
TRAIN_PATH = DATA_PATH + 'train_images/'
TEST_PATH = DATA_PATH + 'test_images/'
print(os.listdir(DATA_PATH))
predictions = reg.predict(x_test )<save_to_csv>
train = pd.read_csv(DATA_PATH + "train.csv")
 test = pd.read_csv(DATA_PATH + "test.csv")
 train["id_code"] = train["id_code"].apply(lambda x: x + ".png")
 test["id_code"] = test["id_code"].apply(lambda x: x + ".png")
 train['diagnosis'] = train['diagnosis'].astype('str')
 train.head(10 )
RFSubmission = pd.DataFrame({ 'key': df_test.key.ravel() ,
'fare_amount': predictions })
 RFSubmission.to_csv("RFSubmission.csv", index=False )<import_modules>
batch_size_train = 32
batch_size_test = 1
train_datagen = ImageDataGenerator(rescale=1./255, validation_split=0.1,
horizontal_flip=True,
vertical_flip=True,
rotation_range=20,
zoom_range= 0.2,
featurewise_center=True,
featurewise_std_normalization=True,
zca_whitening=True,
width_shift_range=0.2,
height_shift_range=0.2,)
 train_gen = train_datagen.flow_from_directory(directory=train_dest,
batch_size= batch_size_train,
class_mode= 'categorical',
target_size=(IMAGE_SIZE, IMAGE_SIZE),
subset='training')
 valid_gen = train_datagen.flow_from_directory(directory=train_dest,
batch_size= batch_size_train,
class_mode= 'categorical',
target_size=(IMAGE_SIZE, IMAGE_SIZE),
subset='validation')
 test_datagen = ImageDataGenerator(rescale=1./255)
 test_gen = test_datagen.flow_from_directory(directory=test_dest,
batch_size= batch_size_test,
class_mode= None,
target_size=(IMAGE_SIZE, IMAGE_SIZE),
shuffle=False )
pd.set_option('display.max_columns', 500)
 pd.set_option('display.max_rows', 500 )<load_from_csv>
def custom_loss(y_true, y_pred):

 custom_cce = math.add(math.multiply(y_true, -1*math.log(0.001+ y_pred*0.999)) ,
math.multiply(1-y_true, -1*math.log(1-y_pred*0.999)))
 custom_cce = math.reduce_sum(custom_cce, 1)
 weights = cast(1 + math.square(math.abs(math.argmax(y_pred, axis=1)-math.argmax(y_true, axis=1)))/100, float32)
 cce = CategoricalCrossentropy()
return math.multiply(cce(y_pred,y_true), weights)
df_train = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv')
 df_test = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv')
 df_data = pd.concat([df_train, df_test])
 df_data.reset_index(drop=True, inplace=True)
 del df_train
del df_test
df_data.info(verbose=False, memory_usage='deep' )<feature_engineering>
print('Creating model...', end=' ')
 input_tensor = Input(shape=(IMAGE_SIZE,IMAGE_SIZE,3))
conv_base = ResNet50(include_top=False, weights=None, input_tensor=input_tensor)
 conv_base.load_weights('.. /input/resnet50/resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5')
 a_map = layers.Conv2D(516, 1, strides=(1, 1), padding="same", activation='relu' )(conv_base.output)
 a_map = layers.Conv2D(1, 1, strides=(1, 1), padding="same", activation='relu' )(a_map)
 a_map = layers.Conv2D(2048, 1, strides=(1, 1), padding="same", activation='sigmoid' )(a_map)
 res = layers.Multiply()([conv_base.output, a_map])
 x = GlobalAveragePooling2D()(res)
 x = Dropout(0.5 )(x)
 x = Dense(2048, activation='relu' )(x)
 x = Dropout(0.5 )(x)
 predictions = Dense(5, activation='softmax', name='final_output' )(x)
 model = Model(input_tensor, predictions)
 model.summary()
print("Done !" )
list_missing_feats = [a for a in df_data.columns if(df_data[a].isnull().sum() > 0)&(not a == 'SalePrice')]
list_numerical_feats = [n for n in df_data.select_dtypes(include=['number'] ).columns]
df_missing = pd.DataFrame(list_missing_feats, columns=['feature'])
 df_missing['feature_count'] = df_missing.apply(lambda x: df_data[x['feature']].isnull().sum() , axis=1)
 df_missing['feature_type'] = np.where(df_missing['feature'].isin(list_numerical_feats), 'number', 'text')
 df_missing = df_missing.sort_values('feature_count' ).reset_index(drop=True)
 df_missing['meh'] = 'x'
df_missing['feature_rank'] = df_missing.sort_values(['feature_count'], ascending=[True] ).groupby(['meh'] ).cumcount()
f, ax = plt.subplots(figsize =(20,8))
sns.set_color_codes('pastel')
 palette = ['r' if(x == 'number')else 'b' for x in df_missing['feature_type']]
z = sns.barplot(x='feature_count', y='feature', data=df_missing, label='Total', palette=palette, edgecolor='w')
 for index, row in df_missing.iterrows() :
text_col = 'b'
if row.feature_type == 'number': text_col = 'r'
z.text(row.feature_count + 15, row.feature_rank, str(row.feature_count), color=text_col, va="center")
 patch_num = mpatches.Patch(color='r', label='Numerical')
 patch_text = mpatches.Patch(color='b', label='Text')
 ax.legend(handles=[patch_num, patch_text])
 sns.despine(left = True, bottom = True)
 plt.show()<data_type_conversions>
train_steps = train_gen.n//train_gen.batch_size
val_steps = valid_gen.n//valid_gen.batch_size
for layer in model.layers:
layer.trainable = False
for i in range(-9, 0):
model.layers[i].trainable = True
optimizer = Adam(lr=1e-4)
 model.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=['accuracy'])
 history_warmup = model.fit_generator(train_gen,
steps_per_epoch=train_steps,
validation_data=valid_gen,
validation_steps=val_steps,
epochs=2 ).history
df_data['Functional'] = df_data['Functional'].fillna('Typ')
 df_data['Utilities'] = df_data['Utilities'].fillna('AllPub')
 for a in ['GarageYrBlt','GarageArea','GarageCars','TotalBsmtSF','MasVnrArea','BsmtFinSF1','BsmtFinSF2','BsmtFullBath','BsmtHalfBath']: df_data[a] = df_data[a].fillna(0)
 for a in ['BsmtExposure','GarageType','GarageFinish','Alley','MiscFeature','BsmtFinType1','BsmtFinType2','BsmtQual','BsmtCond','GarageQual','GarageCond','FireplaceQu','Fence','PoolQC']: df_data[a] = df_data[a].fillna('Na' )<feature_engineering>
for layer in model.layers:
layer.trainable = True
earlystopper = EarlyStopping(monitor='val_loss', patience=7, verbose=1, restore_best_weights=True)
 reducel = ReduceLROnPlateau(monitor='val_loss', patience=2, verbose=1, factor=0.3, min_lr=1e-6)
 optimizer = Adam(1e-4)
 model.compile(loss='categorical_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
 history = model.fit_generator(train_gen,
steps_per_epoch=train_steps,
validation_data=valid_gen,
validation_steps=val_steps,
epochs=25,
callbacks=[reducel, earlystopper] ).history
dict_missing = {}
for b in [a for a in df_data.columns if(df_data[a].isnull().sum() > 0)&(not a == 'SalePrice')]: dict_missing[b] = df_data[b].isnull().sum()
dict_missing = dict(sorted(dict_missing.items() , key=lambda item: item[1]))
for f in dict_missing:
missing_estimator = KNeighborsClassifier(n_neighbors=10)
 if(df_data[f].dtype == np.float64): missing_estimator = KNeighborsRegressor(n_neighbors=10)
 n_cols = [n for n in df_data.select_dtypes(include=['number'] ).columns if(df_data[n].isnull().sum() == 0)&(n not in ['Id','SalePrice'])]
missing_estimator.fit(
 df_data[n_cols][(df_data[f].notnull())],
df_data[f][(df_data[f].notnull())] 
)
 df_data[f] = df_data.apply(lambda x: missing_estimator.predict(np.array(x[n_cols] ).reshape(1,-1)) [0] if pd.isnull(x[f])else x[f], axis=1 )<categorify>
def show_image_mask(path):
im = imread(path)
 exctraction_model = Model(input_tensor, res)
 im = np.array([im])
 w = exctraction_model.predict(im)
 im_res = np.squeeze(im)[:,:,0]
im_w = np.power(resize(np.mean(w[0], axis=2),(224, 224)) ,4)
 imshow(np.multiply(im_w, im_res), cmap = 'gray' )
def _no_trans(df, list_cols, prefix='x_'):
for a in list_cols: df[prefix + a] = df[a]
return df
def _create_dummies(df, list_cols, prefix='x_'):
for a in list_cols:
df_dummies = pd.get_dummies(df[a].astype(str))
df[prefix + a + '_' + df_dummies.columns] = df_dummies
del df_dummies
return df
def _map_val(df, list_cols, prefix='x_'):
for a in list_cols: df[prefix + a] = df[a].fillna('Na' ).map({'Ex':5,'Gd':4,'TA':3,'Fa':2,'Po':1,'Na':0,'GLQ':6,'ALQ':5,'BLQ':4,'Rec':3,'LwQ':2,'Unf':1,'Y':1,'N':0,'GdPrv':4,'MnPrv':3,'GdWo':2,'MnWw':1})
 return df
def _merge_features(df, list_pairs, prefix='x_'):
for a in list_pairs:
list_options = list(set(df[a[0]].to_list() + df[a[1]].to_list()))
df[[prefix + str(a[0])+ '_' + o for o in list_options]] = df.apply(lambda x: _merge_list(list_options, x[a[0]], x[a[1]]), axis=1, result_type='expand')
 return df
def _merge_list(list_data, data_1, data_2):
list_return = [0] * len(list_data)
 list_return[list_data.index(data_1)] = 1
list_return[list_data.index(data_2)] = 1
return list_return
def _rank_features(df, list_cols, prefix='x_rank_'):
for a in list_cols:
ranked_data = df[(df[target].notnull())].groupby(a ).agg({target:'median'} ).sort_values(target ).index
dict_ranks = dict(zip(ranked_data, range(0, len(ranked_data))))
df[prefix + a] = df[a].map(dict_ranks)
 return df<define_variables>
final_valid_datagen = ImageDataGenerator(rescale=1./255)
 final_valid_gen = train_datagen.flow_from_directory(directory= train_dest,
batch_size= 1,
class_mode= 'categorical',
target_size=(IMAGE_SIZE, IMAGE_SIZE),
shuffle=False)
 STEP_SIZE_TEST = final_valid_gen.n//final_valid_gen.batch_size
print('Start predictions...', end=' ')
 preds = model.predict_generator(final_valid_gen, steps=STEP_SIZE_TEST)
 print('Done !')
 predictions = [np.argmax(pred)for pred in preds]
gt= list(final_valid_gen.labels )
list_no_trans = ['LotArea','LotFrontage','MasVnrArea','BsmtFullBath','BsmtHalfBath','BsmtFinSF1','BsmtFinSF2','BsmtUnfSF','TotalBsmtSF','1stFlrSF','GrLivArea','OverallQual','OverallCond','YearBuilt','YearRemodAdd','2ndFlrSF','LowQualFinSF','FullBath','HalfBath','BedroomAbvGr','KitchenAbvGr','TotRmsAbvGrd','Fireplaces','WoodDeckSF','OpenPorchSF','EnclosedPorch','3SsnPorch','ScreenPorch','PoolArea','MiscVal','YrSold','GarageYrBlt','GarageArea','GarageCars']
list_to_dummy = ['MSSubClass','MSZoning','Electrical','MasVnrType','Street','Alley','LotShape','LandContour','LotConfig','LandSlope','Neighborhood','BldgType','HouseStyle','RoofStyle','RoofMatl','Foundation','BsmtExposure','Heating','GarageType','GarageFinish','PavedDrive','MiscFeature','SaleCondition','Functional','Utilities','SaleType']
list_to_map = ['ExterQual','ExterCond','KitchenQual','BsmtQual','BsmtCond','BsmtFinType1','BsmtFinType2','HeatingQC','CentralAir','FireplaceQu','GarageQual','GarageCond','PoolQC','Fence']
list_to_merge = [('Condition1','Condition2'),('Exterior1st','Exterior2nd')]
list_to_rank = ['Heating','HouseStyle','LotShape','MSSubClass','Neighborhood','SaleCondition','SaleType','MasVnrType']
df_data =(df_data 
.pipe(_no_trans, list_no_trans)
.pipe(_create_dummies, list_to_dummy)
.pipe(_map_val, list_to_map)
.pipe(_merge_features, list_to_merge)
.pipe(_rank_features, list_to_rank)
)
 df_data['x_rank_MSSubClass'] = df_data['x_rank_MSSubClass'].fillna(8)
 df_data.info(verbose=False, memory_usage='deep' )<compute_train_metric>
test_gen.reset()
STEP_SIZE_TEST = test_gen.n//test_gen.batch_size
print("Predictions begins...")
 preds = model.predict_generator(test_gen, steps=STEP_SIZE_TEST)
 print("Predictions done !")
 predictions = [np.argmax(pred)for pred in preds]
filenames = test_gen.filenames
results = pd.DataFrame({'id_code':filenames, 'diagnosis':predictions})
 results['id_code'] = results['id_code'].map(lambda x: str(x)[:-4])
 results['id_code'] = results['id_code'].apply(lambda x: x.split('/')[1])
 results.to_csv('submission.csv',index=False)
 results.head(10 )
random_state = 42
def _evaluate(model, X_train, y_train):
cv = KFold(n_splits=4, random_state=random_state, shuffle=True)
 scores = cross_val_score(model, X_train.values, y_train.values, cv=cv, n_jobs=-1, error_score='neg_root_mean_squared_error')
 return scores<create_dataframe>
df = pd.read_csv("submission.csv")
 print(df["diagnosis"].value_counts())
 print(preds )
df_evaluation = pd.DataFrame(columns=['Description','Score','Std'] )<statistical_test>
if os.path.exists('valid/'):
shutil.rmtree('valid/')
 if os.path.exists(train_dest):
shutil.rmtree(train_dest)
 if os.path.exists(test_dest):
shutil.rmtree(test_dest )