Kaggle competition: Understanding Clouds from Satellite Images
<SOS> metric: Dice Kaggle data source: understanding-clouds-from-satellite-images<data_type_conversions>
seed(10)
 set_random_seed(10)
 %matplotlib inline
data['Embarked'].fillna('S', inplace=True)
 data['Embarked'] = data['Embarked'].map({'S': 0, 'C': 1, 'Q': 2} ).astype(int )<categorify>
!pip install keras-rectified-adam
data['Title'] = data['Name'].str.split(', ', expand=True)[1].str.split('.', expand=True)[0]
data['Title'] = data['Title'].replace(['Miss', 'Mrs','Ms', 'Mlle', 'Lady', 'Mme', 'the Countess', 'Dona'], 'Miss/Mrs/Ms')
 data['Title'] = data['Title'].replace(['Dr', 'Col', 'Major', 'Jonkheer', 'Capt', 'Sir', 'Don'], 'Dr/Military/Noble')
 data['Title'] = data['Title'].fillna('Mr' ).map({'Mr': 0, 'Mrs': 1, 'Miss/Mrs/Ms': 2, 'Dr/Military/Noble': 3, 'Master': 4, 'Rev': 5} ).astype(int )<count_values>
test_imgs_folder = '.. /input/understanding_cloud_organization/test_images/'
train_imgs_folder = '.. /input/understanding_cloud_organization/train_images/'
num_cores = multiprocessing.cpu_count()
data['Title'].value_counts()<drop_column>
train_df = pd.read_csv('.. /input/understanding_cloud_organization/train.csv')
 train_df.head()
drop_columns = ["PassengerId", "Ticket","Cabin", "Name", "SibSp"]
data = data.drop(drop_columns, axis=1 )<count_missing_values>
train_df = train_df[~train_df['EncodedPixels'].isnull() ]
train_df['Image'] = train_df['Image_Label'].map(lambda x: x.split('_')[0])
 train_df['Class'] = train_df['Image_Label'].map(lambda x: x.split('_')[1])
 classes = train_df['Class'].unique()
train_df = train_df.groupby('Image')['Class'].agg(set ).reset_index()
for class_name in classes:
train_df[class_name] = train_df['Class'].map(lambda x: 1 if class_name in x else 0)
 train_df.head()
data.isnull().sum()<drop_column>
img_2_ohe_vector = {img:vec for img, vec in zip(train_df['Image'], train_df.iloc[:, 2:].values)}
train = data[data['split'] == 'train'].copy()
test = data[data['split'] == 'test'].copy()
train_labels = train["Survived"].copy()
train.drop(['split','Survived'], axis=1, inplace=True)
 test.drop(['split','Survived'], axis=1, inplace=True )<split>
train_imgs, val_imgs = train_test_split(train_df['Image'].values,
test_size=0.2,
stratify=train_df['Class'].map(lambda x: str(sorted(list(x)))) ,
random_state=2019 )
train, train_val, train_labels, train_val_labels = train_test_split(train, train_labels, test_size=0.2, random_state=42 )<choose_model_class>
class DataGenenerator(Sequence):
def __init__(self, images_list=None, folder_imgs=train_imgs_folder,
batch_size=32, shuffle=True, augmentation=None,
resized_height=260, resized_width=260, num_channels=3):
self.batch_size = batch_size
self.shuffle = shuffle
self.augmentation = augmentation
if images_list is None:
self.images_list = os.listdir(folder_imgs)
 else:
self.images_list = deepcopy(images_list)
 self.folder_imgs = folder_imgs
self.len = len(self.images_list)// self.batch_size
self.resized_height = resized_height
self.resized_width = resized_width
self.num_channels = num_channels
self.num_classes = 4
self.is_test = not 'train' in folder_imgs
if not shuffle and not self.is_test:
self.labels = [img_2_ohe_vector[img] for img in self.images_list[:self.len*self.batch_size]]
def __len__(self):
return self.len
def on_epoch_start(self):
if self.shuffle:
random.shuffle(self.images_list)
 def __getitem__(self, idx):
current_batch = self.images_list[idx * self.batch_size:(idx + 1)* self.batch_size]
X = np.empty(( self.batch_size, self.resized_height, self.resized_width, self.num_channels))
y = np.empty(( self.batch_size, self.num_classes))
for i, image_name in enumerate(current_batch):
path = os.path.join(self.folder_imgs, image_name)
 img = cv2.resize(cv2.imread(path),(self.resized_height, self.resized_width)).astype(np.float32)
 if not self.augmentation is None:
augmented = self.augmentation(image=img)
 img = augmented['image']
X[i, :, :, :] = img/255.0
if not self.is_test:
y[i, :] = img_2_ohe_vector[image_name]
return X, y
def get_labels(self):
if self.shuffle:
images_current = self.images_list[:self.len*self.batch_size]
labels = [img_2_ohe_vector[img] for img in images_current]
else:
labels = self.labels
return np.array(labels )
num_pipeline = Pipeline([ 
("imputer", SimpleImputer(strategy="median")) , 
("MinMaxScaler", MinMaxScaler())
 ] )<normalization>
albumentations_train = Compose([
VerticalFlip() , HorizontalFlip() , Rotate(limit=20), GridDistortion()
], p=1 )
train_prepared = num_pipeline.fit_transform(train)
 train_val = num_pipeline.fit_transform(train_val )<train_on_grid>
data_generator_train = DataGenenerator(train_imgs, augmentation=albumentations_train)
 data_generator_train_eval = DataGenenerator(train_imgs, shuffle=False)
 data_generator_val = DataGenenerator(val_imgs, shuffle=False )
model_lgre = LogisticRegression(random_state=0)
 param_grid = {'C': [0.014,0.012], 'multi_class': ['multinomial'],
'penalty': ['l1'],'solver': ['saga'], 'tol': [0.1] }
GridCV_LR = GridSearchCV(model_lgre, param_grid, verbose=1, cv=5)
 GridCV_LR.fit(train_prepared,train_labels)
 score_grid_LR = GridCV_LR.best_score_
model_lgre = GridCV_LR.best_estimator_
print(score_grid_LR)
 predict_score_lg_clf = GridCV_LR.predict(train_val)
 report_and_confusion_matrix(train_val_labels, predict_score_lg_clf )<train_on_grid>
class PrAucCallback(Callback):
def __init__(self, data_generator, num_workers=num_cores,
early_stopping_patience=5,
plateau_patience=3, reduction_rate=0.5,
stage='train', checkpoints_path='checkpoints/'):
super(Callback, self ).__init__()
self.data_generator = data_generator
self.num_workers = num_workers
self.class_names = ['Fish', 'Flower', 'Sugar', 'Gravel']
self.history = [[] for _ in range(len(self.class_names)+ 1)]
self.early_stopping_patience = early_stopping_patience
self.plateau_patience = plateau_patience
self.reduction_rate = reduction_rate
self.stage = stage
self.best_pr_auc = -float('inf')
 if not os.path.exists(checkpoints_path):
os.makedirs(checkpoints_path)
 self.checkpoints_path = checkpoints_path
def compute_pr_auc(self, y_true, y_pred):
pr_auc_mean = 0
print(f"
{'
")
 for class_i in range(len(self.class_names)) :
precision, recall, _ = precision_recall_curve(y_true[:, class_i], y_pred[:, class_i])
 pr_auc = auc(recall, precision)
 pr_auc_mean += pr_auc/len(self.class_names)
 print(f"PR AUC {self.class_names[class_i]}, {self.stage}: {pr_auc:.3f}
")
 self.history[class_i].append(pr_auc)
 print(f"
{'
PR AUC mean, {self.stage}: {pr_auc_mean:.3f}
{'
")
 self.history[-1].append(pr_auc_mean)
 return pr_auc_mean
def is_patience_lost(self, patience):
if len(self.history[-1])> patience:
best_performance = max(self.history[-1][-(patience + 1):-1])
 return best_performance == self.history[-1][-(patience + 1)] and best_performance >= self.history[-1][-1]
def early_stopping_check(self, pr_auc_mean):
if self.is_patience_lost(self.early_stopping_patience):
self.model.stop_training = True
def model_checkpoint(self, pr_auc_mean, epoch):
if pr_auc_mean > self.best_pr_auc:
for checkpoint in glob.glob(os.path.join(self.checkpoints_path, 'classifier_densenet169_epoch_*')) :
os.remove(checkpoint)
 self.best_pr_auc = pr_auc_mean
self.model.save(os.path.join(self.checkpoints_path, f'classifier_densenet169_epoch_{epoch}_val_pr_auc_{pr_auc_mean}.h5'))
print(f"
{'
Saved new checkpoint
{'
")
 def reduce_lr_on_plateau(self):
if self.is_patience_lost(self.plateau_patience):
new_lr = float(keras.backend.get_value(self.model.optimizer.lr)) * self.reduction_rate
keras.backend.set_value(self.model.optimizer.lr, new_lr)
 print(f"
{'
Reduced learning rate to {new_lr}.
 {'
")
 def on_epoch_end(self, epoch, logs={}):
y_pred = self.model.predict_generator(self.data_generator, workers=self.num_workers)
 y_true = self.data_generator.get_labels()
pr_auc_mean = self.compute_pr_auc(y_true, y_pred)
 if self.stage == 'val':
self.early_stopping_check(pr_auc_mean)
 self.model_checkpoint(pr_auc_mean, epoch)
 self.reduce_lr_on_plateau()
def get_pr_auc_history(self):
return self.history
rand_forest_clf = RandomForestClassifier(random_state=42)
 param_grid = {'max_depth': [15], 'min_samples_split': [5],'n_estimators' : [100] }
GridCV_rd_clf = GridSearchCV(rand_forest_clf, param_grid, verbose=1, cv=5)
 GridCV_rd_clf.fit(train_prepared, train_labels)
 score_grid_rd = GridCV_rd_clf.best_score_
rand_forest_clf = GridCV_rd_clf.best_estimator_
print(score_grid_rd)
 predict_score_rd_clf = GridCV_rd_clf.predict(train_val)
 report_and_confusion_matrix(train_val_labels, predict_score_rd_clf )<choose_model_class>
train_metric_callback = PrAucCallback(data_generator_train_eval)
 val_callback = PrAucCallback(data_generator_val, stage='val' )
tree_clf = DecisionTreeClassifier()<train_on_grid>
def dice_coef(y_true, y_pred, smooth=1):
y_true_f = K.flatten(y_true)
 y_pred_f = K.flatten(y_pred)
 intersection = K.sum(y_true_f * y_pred_f)
 return(2.* intersection + smooth)/(K.sum(y_true_f)+ K.sum(y_pred_f)+ smooth)
 def dice_loss(y_true, y_pred):
smooth = 1.
 y_true_f = K.flatten(y_true)
 y_pred_f = K.flatten(y_pred)
 intersection = y_true_f * y_pred_f
score =(2.* K.sum(intersection)+ smooth)/(K.sum(y_true_f)+ K.sum(y_pred_f)+ smooth)
 return 1.- score
def bce_dice_loss(y_true, y_pred):
return binary_crossentropy(y_true, y_pred)+ dice_loss(y_true, y_pred )
params_grid_ada = {}
ada_model = AdaBoostClassifier(tree_clf,n_estimators=3000,
algorithm="SAMME.R", learning_rate=0.05, random_state=42)
 GridCV_ada = GridSearchCV(ada_model, params_grid_ada, verbose=1, cv=5)
 GridCV_ada.fit(train_prepared, train_labels)
 score_grid_ada = GridCV_ada.best_score_
model_ada = GridCV_ada.best_estimator_
print(score_grid_ada)
 predict_score_ada = GridCV_ada.predict(train_val)
 report_and_confusion_matrix(train_val_labels, predict_score_ada )<choose_model_class>
!pip install -U git+https://github.com/qubvel/efficientnet
params_grid_gb = {}
gbc_model = GradientBoostingClassifier(criterion='friedman_mse', init=None,
learning_rate=0.05, loss='deviance', max_depth=3,
max_features=None, max_leaf_nodes=None,
min_impurity_decrease=0.0, min_impurity_split=None,
min_samples_leaf=1, min_samples_split=2,
min_weight_fraction_leaf=0.0, n_estimators=10000,
n_iter_no_change=None, presort='auto', random_state=None,
subsample=1.0, tol=0.0001, validation_fraction=0.1,
verbose=0, warm_start=False)
 GridCV_GB = GridSearchCV(gbc_model, params_grid_gb, verbose=1, cv=5)
 GridCV_GB.fit(train_prepared, train_labels)
 score_grid_GB = GridCV_GB.best_score_
model_gbc = GridCV_GB.best_estimator_
print(score_grid_GB)
 predict_score_GB = GridCV_GB.predict(train_val)
 report_and_confusion_matrix(train_val_labels, predict_score_GB )<train_on_grid>
def get_model() :
K.clear_session()
base_model = efn.EfficientNetB2(weights='imagenet', include_top=False, pooling='avg', input_shape=(260, 260, 3))
x = base_model.output
y_pred = Dense(4, activation='sigmoid' )(x)
 return Model(inputs=base_model.input, outputs=y_pred)
 model = get_model()
mlp_clf = MLPClassifier(activation = "logistic", hidden_layer_sizes=(300,), random_state=42,batch_size=1000)
 param_grid = { 'max_iter': [1200], 'alpha': [1e-4],
'solver': ['sgd'], 'learning_rate_init': [0.05,0.06],'tol': [1e-4] }
GridCV_MLP = GridSearchCV(mlp_clf, param_grid, verbose=1, cv=3)
 GridCV_MLP.fit(train,train_labels)
 score_grid_MLP = GridCV_MLP.best_score_
model_mlp = GridCV_MLP.best_estimator_
print(score_grid_MLP)
 predict_score_MLP = GridCV_MLP.predict(train_val)
 report_and_confusion_matrix(train_val_labels, predict_score_MLP )<choose_model_class>
from keras_radam import RAdam
params_grid_search_lgb = {
}
lgb_model = lgb.LGBMClassifier(boosting_type='gbdt', class_weight=None, colsample_bytree=0.7,
gamma=0, importance_type='split', learning_rate=0.05, max_depth=3,
min_child_samples=20, min_child_weight=6, min_split_gain=0.0,
n_estimators=20000, n_jobs=-1, nthread=4, num_leaves=31,
objective=None, random_state=None, reg_alpha=0.0, reg_lambda=0.0,
scale_pos_weight=1, seed=29, silent=True, subsample=0.7,
subsample_for_bin=200000, subsample_freq=0)
 GridCV_LGB = GridSearchCV(lgb_model, params_grid_search_lgb, verbose=1, cv=5)
 GridCV_LGB.fit(train_prepared,train_labels)
 score_grid_LGB = GridCV_LGB.best_score_
model_lgb = GridCV_LGB.best_estimator_
print(score_grid_LGB)
 predict_score_LGB = GridCV_LGB.predict(train_val)
 report_and_confusion_matrix(train_val_labels, predict_score_LGB )<train_on_grid>
for base_layer in model.layers[:-3]:
base_layer.trainable = False
model.compile(optimizer=RAdam(warmup_proportion=0.1, min_lr=1e-5), loss='categorical_crossentropy', metrics=['accuracy'])
 history_0 = model.fit_generator(generator=data_generator_train,
validation_data=data_generator_val,
epochs=20,
callbacks=[train_metric_callback, val_callback],
workers=num_cores,
verbose=1
)
params_grid_xgb = {}
xgb_model = xgb.XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,
colsample_bytree=0.7, gamma=0.2, learning_rate=0.009,
max_delta_step=0, max_depth=3, min_child_weight=6, missing=None,
n_estimators=10000, n_jobs=1, nthread=4, objective='binary:logistic',
random_state=0, reg_alpha=0, reg_lambda=1, scale_pos_weight=1,
seed=29, silent=True, subsample=0.7)
 GridCV_XGB = GridSearchCV(xgb_model, params_grid_xgb, verbose=1, cv=5)
 GridCV_XGB.fit(train_prepared, train_labels)
 score_grid_XGB = GridCV_XGB.best_score_
model_xgb = GridCV_XGB.best_estimator_
print(score_grid_XGB)
 predict_score_XGB = GridCV_XGB.predict(train_val)
 report_and_confusion_matrix(train_val_labels, predict_score_XGB )<train_on_grid>
for base_layer in model.layers[:-3]:
base_layer.trainable = True
model.compile(optimizer=RAdam(warmup_proportion=0.1, min_lr=1e-5), loss='categorical_crossentropy', metrics=['accuracy'])
 history_1 = model.fit_generator(generator=data_generator_train,
validation_data=data_generator_val,
epochs=20,
callbacks=[train_metric_callback, val_callback],
workers=num_cores,
verbose=1,
initial_epoch=1
)
params_grid_vclf = {}
estimators = [ 
("Ada Boosting", model_ada), 
("Gradient Boost", model_gbc), 
("lgb_model", model_lgb), 
("XGBoost", model_xgb), 
("MLP", model_mlp),
]
voting_clf = VotingClassifier(estimators,voting='soft')
 GridCV_voting_clf = GridSearchCV(voting_clf, params_grid_vclf, verbose=1, cv=5)
 GridCV_voting_clf.fit(train_prepared, train_labels)
 score_grid_vclf = GridCV_voting_clf.best_score_
model_vclf = GridCV_voting_clf.best_estimator_
print(score_grid_vclf)
 predict_score_vclf = GridCV_voting_clf.predict(train_val)
 report_and_confusion_matrix(train_val_labels, predict_score_vclf)
<categorify>
Image(".. /input/clouds-classifier-files/loss_hist_densenet169.png" )
test_prepared = num_pipeline.fit_transform(test )<predict_on_test>
Image(".. /input/clouds-classifier-files/pr_auc_hist_densenet169.png" )
sub_voting_classifier = model_vclf.predict(test_prepared )<choose_model_class>
class_names = ['Fish', 'Flower', 'Sugar', 'Gravel']
def get_threshold_for_recall(y_true, y_pred, class_i, recall_threshold=0.94, precision_threshold=0.90, plot=False):
precision, recall, thresholds = precision_recall_curve(y_true[:, class_i], y_pred[:, class_i])
 i = len(thresholds)- 1
best_recall_threshold = None
while best_recall_threshold is None:
next_threshold = thresholds[i]
next_recall = recall[i]
if next_recall >= recall_threshold:
best_recall_threshold = next_threshold
i -= 1
best_precision_threshold = [thres for prec, thres in zip(precision, thresholds)if prec >= precision_threshold][0]
if plot:
plt.figure(figsize=(10, 7))
plt.step(recall, precision, color='r', alpha=0.3, where='post')
 plt.fill_between(recall, precision, alpha=0.3, color='r')
 plt.axhline(y=precision[i + 1])
 recall_for_prec_thres = [rec for rec, thres in zip(recall, thresholds)
 if thres == best_precision_threshold][0]
plt.axvline(x=recall_for_prec_thres, color='g')
 plt.xlabel('Recall')
 plt.ylabel('Precision')
 plt.ylim([0.0, 1.05])
 plt.xlim([0.0, 1.0])
 plt.legend(['PR curve',
f'Precision {precision[i + 1]:.2f} corresponding to selected recall threshold',
f'Recall {recall_for_prec_thres:.2f} corresponding to selected precision threshold'])
 plt.title(f'Precision-Recall curve for Class {class_names[class_i]}')
 return best_recall_threshold, best_precision_threshold
y_pred = model.predict_generator(data_generator_val, workers=num_cores)
 y_true = data_generator_val.get_labels()
recall_thresholds = dict()
precision_thresholds = dict()
for i, class_name in tqdm(enumerate(class_names)) :
recall_thresholds[class_name], precision_thresholds[class_name] = get_threshold_for_recall(y_true, y_pred, i, plot=True )
estimators =[model_ada,model_lgb,model_gbc, model_mlp]
sclf = StackingCVClassifier(classifiers= estimators,
meta_classifier=model_xgb)
 print('3-fold cross validation:
')
 for clf, label in zip([model_ada, model_lgb, model_gbc, model_mlp, sclf],
['Ada Boosting',
'light gradient Boosting(LGBM)',
'Gradient Boosting',
"MLP",
'StackingClassifier(Meta: XGboost)']):
scores = model_selection.cross_val_score(clf, train_prepared, train_labels,
cv=5, scoring='precision')
 print("precision: %0.2f(+/- %0.2f)[%s]"
%(scores.mean() , scores.std() , label))<train_model>
data_generator_test = DataGenenerator(folder_imgs=test_imgs_folder, shuffle=False)
 y_pred_test = model.predict_generator(data_generator_test, workers=num_cores )
sclf.fit(train_prepared, train_labels )<predict_on_test>
image_labels_empty = set()
for i,(img, predictions)in enumerate(zip(os.listdir(test_imgs_folder), y_pred_test)) :
for class_i, class_name in enumerate(class_names):
if predictions[class_i] < recall_thresholds[class_name]:
image_labels_empty.add(f'{img}_{class_name}' )
prediction_sclf = sclf.predict(train_val)
 report_and_confusion_matrix(train_val_labels, prediction_sclf )<predict_on_test>
submission = pd.read_csv('.. /input/densenet201cloudy/densenet201.csv')
 submission.head()
prediction_stacking_clf = sclf.predict(test_prepared )<save_to_csv>
predictions_nonempty = set(submission.loc[~submission['EncodedPixels'].isnull() , 'Image_Label'].values )
sub = pd.read_csv('.. /input/gender_submission.csv')
 sub['Survived'] = sub_voting_classifier.astype('int64')
 sub.to_csv("voting_classifier.csv", index=False)
 sub1 = pd.read_csv('.. /input/gender_submission.csv')
 sub1['Survived'] = prediction_stacking_clf.astype('int64')
 sub.to_csv("stacking_clf.csv", index=False )<import_modules>
print(f'{len(image_labels_empty.intersection(predictions_nonempty)) } masks would be removed' )
<choose_model_class><EOS> 
submission.loc[submission['Image_Label'].isin(image_labels_empty), 'EncodedPixels'] = np.nan
submission.to_csv('submission_segmentation_and_classifier.csv', index=None )