Kaggle competition: Understanding Clouds from Satellite Images
<SOS> metric: Dice Kaggle data source: understanding-clouds-from-satellite-images<data_type_conversions>
! pip install git+https://github.com/qubvel/classification_models.git
raw_data['IsAgeNull'] = np.where(np.isnan(raw_data['Age']), 1, 0)
 raw_data['Age'].fillna(( raw_data['Age'].mean()), inplace=True)
 raw_data['Age'] = raw_data['Age'].round().astype(int )<feature_engineering>
seed(10)
 set_random_seed(10)
 %matplotlib inline
raw_data['AgeLabel'] = pd.cut(raw_data['Age'], bins=np.arange(start=0, stop=90, step=10), labels=np.arange(start=0, stop=8, step=1), include_lowest=True )<feature_engineering>
from classification_models.keras import Classifiers
raw_data['FareLabel'] = pd.cut(raw_data.Fare, bins=np.arange(start=0, stop=600, step=50), precision=0, labels=np.arange(start=0, stop=11, step=1), include_lowest=True )<categorify>
test_imgs_folder = '.. /input/understanding_cloud_organization/test_images/'
train_imgs_folder = '.. /input/understanding_cloud_organization/train_images/'
num_cores = multiprocessing.cpu_count()
raw_data = pd.concat([raw_data, pd.get_dummies(raw_data['Sex'])], axis=1 )<categorify>
train_df = pd.read_csv('.. /input/understanding_cloud_organization/train.csv')
 train_df.head()
raw_data = pd.concat([raw_data, pd.get_dummies(raw_data['Title'], prefix='title')], axis=1 )<categorify>
train_df = train_df[~train_df['EncodedPixels'].isnull() ]
train_df['Image'] = train_df['Image_Label'].map(lambda x: x.split('_')[0])
 train_df['Class'] = train_df['Image_Label'].map(lambda x: x.split('_')[1])
 classes = train_df['Class'].unique()
train_df = train_df.groupby('Image')['Class'].agg(set ).reset_index()
for class_name in classes:
train_df[class_name] = train_df['Class'].map(lambda x: 1 if class_name in x else 0)
 train_df.head()
raw_data = pd.concat([raw_data, pd.get_dummies(raw_data['Pclass'], prefix='Pclass')], axis=1 )<prepare_x_and_y>
img_2_ohe_vector = {img:vec for img, vec in zip(train_df['Image'], train_df.iloc[:, 2:].values)}
need_columns = [
'female', 'male',
'Pclass_1', 'Pclass_2', 'Pclass_3',
'title_Master', 'title_Miss', 'title_Mr', 'title_Mrs', 'title_Rare',
'AgeLabel', 'IsAgeNull', 'IsChildren',
'FareLabel', 'SibSp', 'Parch', 'FamilySize',
'NoFamily', 'SmallFamily', 'MediumFamily', 'LargeFamily',
'LowFare', 'HighFare', 'MediumFare',
'NoCabin'
]
data = raw_data[need_columns]
x = data
y = raw_data.Survived 
<define_search_space>
albumentations_train = Compose([
VerticalFlip() , HorizontalFlip() , Rotate(limit=30), GridDistortion()
], p=1 )
n_estimators = [int(x)for x in np.linspace(start = 100, stop = 2000, num = 20)]
max_features = ['auto', 'sqrt', 'log2', None]
max_depth = [int(x)for x in np.linspace(10, 200, num = 20)]
max_depth.append(None)
 min_samples_split = [2, 5, 10]
min_samples_leaf = [1, 2, 4]
bootstrap = [True, False]
max_leaf_nodes = [2, 5, 8, 10, None]
criterion=['gini', 'entropy']
random_grid = {
'n_estimators': n_estimators,
'criterion': criterion,
'max_features': max_features,
'max_depth': max_depth,
'min_samples_split': min_samples_split,
'min_samples_leaf': min_samples_leaf,
'bootstrap': bootstrap,
'max_leaf_nodes': max_leaf_nodes
}
estimator = RandomForestClassifier()<train_on_grid>
train_imgs, val_imgs = train_test_split(train_df['Image'].values,
test_size=0.2,
stratify=train_df['Class'].map(lambda x: str(sorted(list(x)))) ,
random_state=10 )
rf_random = RandomizedSearchCV(
 estimator=estimator,
param_distributions=random_grid,
random_state=42,
n_jobs=-1 
)
 rf_random.fit(x, y )<find_best_params>
class DataGenenerator(Sequence):
def __init__(self, images_list=None, folder_imgs=train_imgs_folder,
batch_size=8, shuffle=True, augmentation=None,
resized_height=224, resized_width=224, num_channels=3):
self.batch_size = batch_size
self.shuffle = shuffle
self.augmentation = augmentation
if images_list is None:
self.images_list = os.listdir(folder_imgs)
 else:
self.images_list = deepcopy(images_list)
 self.folder_imgs = folder_imgs
self.len = len(self.images_list)// self.batch_size
self.resized_height = resized_height
self.resized_width = resized_width
self.num_channels = num_channels
self.num_classes = 4
self.is_test = not 'train' in folder_imgs
if not shuffle and not self.is_test:
self.labels = [img_2_ohe_vector[img] for img in self.images_list[:self.len*self.batch_size]]
def __len__(self):
return self.len
def on_epoch_start(self):
if self.shuffle:
random.shuffle(self.images_list)
 def __getitem__(self, idx):
current_batch = self.images_list[idx * self.batch_size:(idx + 1)* self.batch_size]
X = np.empty(( self.batch_size, self.resized_height, self.resized_width, self.num_channels))
y = np.empty(( self.batch_size, self.num_classes))
for i, image_name in enumerate(current_batch):
path = os.path.join(self.folder_imgs, image_name)
 img = cv2.resize(cv2.imread(path),(self.resized_height, self.resized_width)).astype(np.float32)
 if not self.augmentation is None:
augmented = self.augmentation(image=img)
 img = augmented['image']
X[i, :, :, :] = img/255.0
if not self.is_test:
y[i, :] = img_2_ohe_vector[image_name]
return X, y
def get_labels(self):
if self.shuffle:
images_current = self.images_list[:self.len*self.batch_size]
labels = [img_2_ohe_vector[img] for img in images_current]
else:
labels = self.labels
return np.array(labels )
best_params = rf_random.best_params_
print(best_params )<find_best_score>
data_generator_train = DataGenenerator(train_imgs, augmentation=albumentations_train)
 data_generator_train_eval = DataGenenerator(train_imgs, shuffle=False)
 data_generator_val = DataGenenerator(val_imgs, shuffle=False )
best_score = rf_random.best_score_
print("best score {}".format(best_score))<choose_model_class>
class PrAucCallback(Callback):
def __init__(self, data_generator, num_workers=num_cores,
early_stopping_patience=5,
plateau_patience=3, reduction_rate=0.5,
stage='train', checkpoints_path='checkpoints/'):
super(Callback, self ).__init__()
self.data_generator = data_generator
self.num_workers = num_workers
self.class_names = ['Fish', 'Flower', 'Sugar', 'Gravel']
self.history = [[] for _ in range(len(self.class_names)+ 1)]
self.early_stopping_patience = early_stopping_patience
self.plateau_patience = plateau_patience
self.reduction_rate = reduction_rate
self.stage = stage
self.best_pr_auc = -float('inf')
 if not os.path.exists(checkpoints_path):
os.makedirs(checkpoints_path)
 self.checkpoints_path = checkpoints_path
def compute_pr_auc(self, y_true, y_pred):
pr_auc_mean = 0
print(f"
{'
")
 for class_i in range(len(self.class_names)) :
precision, recall, _ = precision_recall_curve(y_true[:, class_i], y_pred[:, class_i])
 pr_auc = auc(recall, precision)
 pr_auc_mean += pr_auc/len(self.class_names)
 print(f"PR AUC {self.class_names[class_i]}, {self.stage}: {pr_auc:.3f}
")
 self.history[class_i].append(pr_auc)
 print(f"
{'
PR AUC mean, {self.stage}: {pr_auc_mean:.3f}
{'
")
 self.history[-1].append(pr_auc_mean)
 return pr_auc_mean
def is_patience_lost(self, patience):
if len(self.history[-1])> patience:
best_performance = max(self.history[-1][-(patience + 1):-1])
 return best_performance == self.history[-1][-(patience + 1)] and best_performance >= self.history[-1][-1]
def early_stopping_check(self, pr_auc_mean):
if self.is_patience_lost(self.early_stopping_patience):
self.model.stop_training = True
def model_checkpoint(self, pr_auc_mean, epoch):
if pr_auc_mean > self.best_pr_auc:
for checkpoint in glob.glob(os.path.join(self.checkpoints_path, 'classifier_densenet169_epoch_*')) :
os.remove(checkpoint)
 self.best_pr_auc = pr_auc_mean
self.model.save(os.path.join(self.checkpoints_path, f'classifier_densenet169_epoch_{epoch}_val_pr_auc_{pr_auc_mean}.h5'))
print(f"
{'
Saved new checkpoint
{'
")
 def reduce_lr_on_plateau(self):
if self.is_patience_lost(self.plateau_patience):
new_lr = float(keras.backend.get_value(self.model.optimizer.lr)) * self.reduction_rate
keras.backend.set_value(self.model.optimizer.lr, new_lr)
 print(f"
{'
Reduced learning rate to {new_lr}.
 {'
")
 def on_epoch_end(self, epoch, logs={}):
y_pred = self.model.predict_generator(self.data_generator, workers=self.num_workers)
 y_true = self.data_generator.get_labels()
pr_auc_mean = self.compute_pr_auc(y_true, y_pred)
 if self.stage == 'val':
self.early_stopping_check(pr_auc_mean)
 self.model_checkpoint(pr_auc_mean, epoch)
 self.reduce_lr_on_plateau()
def get_pr_auc_history(self):
return self.history
test_model = RandomForestClassifier(
 n_estimators=best_params['n_estimators'],
criterion=best_params['criterion'],
max_features=best_params['max_features'],
max_depth=best_params['max_depth'],
min_samples_split=best_params['min_samples_split'],
min_samples_leaf=best_params['min_samples_leaf'],
max_leaf_nodes=best_params['max_leaf_nodes'],
bootstrap=best_params['bootstrap']
)<compute_train_metric>
def get_model() :
K.clear_session()
seresnext101, preprocess_input = Classifiers.get('seresnext101')
 base_model = seresnext101(weights='imagenet', include_top=False, input_shape=(224, 224, 3))
x = base_model.output
x = keras.layers.GlobalAveragePooling2D()(x)
 x = Dense(512 )(x)
 x = BatchNormalization()(x)
 x = LeakyReLU()(x)
 y_pred = Dense(4, activation='sigmoid' )(x)
 return Model(inputs=base_model.input, outputs=y_pred )
scores = cross_val_score(test_model, x, y, cv=10, scoring='accuracy' )<compute_test_metric>
model = get_model()
print(scores)
 print("Mean Accuracy: {}".format(scores.mean()))<train_model>
for base_layer in model.layers[:-1]:
base_layer.trainable = False
lr=1e-3
adam = Adam(lr=1e-3)
 model.compile(optimizer=adam, loss='binary_crossentropy', metrics=["accuracy"])
 history_0 = model.fit_generator(generator=data_generator_train,
validation_data=data_generator_val,
epochs=1,
workers=num_cores, use_multiprocessing=True,
verbose=1
)
model = RandomForestClassifier(
 n_estimators=best_params['n_estimators'],
criterion=best_params['criterion'],
max_features=best_params['max_features'],
max_depth=best_params['max_depth'],
min_samples_split=best_params['min_samples_split'],
min_samples_leaf=best_params['min_samples_leaf'],
max_leaf_nodes=best_params['max_leaf_nodes'],
bootstrap=best_params['bootstrap'] 
)
 model.fit(x, y )<count_missing_values>
for base_layer in model.layers[:-1]:
base_layer.trainable = True
lr=lr/3.0
K.set_value(adam.lr, lr)
 model.compile(optimizer=adam, loss='binary_crossentropy', metrics=["accuracy"])
 print("training...")
 history_1 = model.fit_generator(generator=data_generator_train,
validation_data=data_generator_val,
epochs=20, use_multiprocessing=True,
workers=num_cores,
verbose=1 
)
 model.save("model.h5" )
print(raw_test.isnull().sum())
 print("-"*10)
 print(raw_test.isnull().sum() /raw_test.shape[0] )<feature_engineering>
lr=lr/3.0
K.set_value(adam.lr, lr)
 history_1 = model.fit_generator(generator=data_generator_train,
validation_data=data_generator_val,
epochs=15, use_multiprocessing=True,
workers=num_cores,
verbose=1 
)
 model.save("model.h5" )
raw_test['Fare'].fillna(raw_test['Fare'].mean() , inplace=True)
 raw_test['NoCabin'] = np.where(raw_test['Cabin'].isnull() , 1, 0)
 raw_test['IsChildren'] = np.where(raw_test['Age']<=10, 1, 0)
 raw_test['IsAgeNull'] = np.where(np.isnan(raw_test['Age']), 1, 0)
 raw_test['Age'].fillna(raw_test['Age'].mean() , inplace=True)
 raw_test['Age'] = raw_test['Age'].round().astype(int)
 raw_test['AgeLabel'] = pd.cut(raw_test['Age'], bins=np.arange(start=0, stop=90, step=10), labels=np.arange(start=0, stop=8, step=1), include_lowest=True)
 raw_test['FamilySize'] = raw_test.apply(lambda row: row['SibSp']+row['Parch'], axis=1)
 raw_test['LowFare'] = np.where(raw_test['Fare']<=50, 1, 0)
 raw_test['HighFare'] = np.where(raw_test['Fare']>300, 1, 0)
 raw_test['MediumFare'] = raw_test.apply(medium_fare, axis=1)
 raw_test['FareLabel'] = pd.cut(raw_test.Fare, bins=np.arange(start=0, stop=600, step=50), precision=0, labels=np.arange(start=0, stop=11, step=1), include_lowest=True)
 raw_test['NoFamily'] = np.where(raw_test['FamilySize']==0, 1, 0)
 raw_test['SmallFamily'] = np.where(( raw_test['FamilySize']>0)&(raw_test['FamilySize']<4), 1, 0)
 raw_test['MediumFamily'] = np.where(( raw_test['FamilySize']>3)&(raw_test['FamilySize']<7), 1, 0)
 raw_test['LargeFamily'] = np.where(raw_test['FamilySize']>=7, 1, 0)
 raw_test['Title'] = raw_test.Name.str.extract('([A-Za-z]+)\.', expand=False)
 raw_test['Title'] = raw_test['Title'].replace(['Lady', 'Countess','Capt', 'Col','Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')
 raw_test['Title'] = raw_test['Title'].replace('Mlle', 'Miss')
 raw_test['Title'] = raw_test['Title'].replace('Ms', 'Miss')
 raw_test['Title'] = raw_test['Title'].replace('Mme', 'Mrs')
 raw_test = pd.concat([raw_test, pd.get_dummies(raw_test['Sex'])], axis=1)
 raw_test = pd.concat([raw_test, pd.get_dummies(raw_test['Title'], prefix='title')], axis=1)
 raw_test = pd.concat([raw_test, pd.get_dummies(raw_test['Pclass'], prefix='Pclass')], axis=1)
 data_test = raw_test[need_columns] 
<save_to_csv>
lr=lr/3.0
K.set_value(adam.lr, lr)
 history_1 = model.fit_generator(generator=data_generator_train,
validation_data=data_generator_val,
epochs=10, use_multiprocessing=True,
workers=num_cores,
verbose=1 
)
 model.save("model.h5" )
ids = raw_test['PassengerId']
predictions = model.predict(data_test)
 output = pd.DataFrame({ 'PassengerId' : ids, 'Survived': predictions })
 output.to_csv('submission.csv', index = False)
 output.head(10 )<load_from_csv>
lr=lr/3.0
K.set_value(adam.lr, lr)
 history_1 = model.fit_generator(generator=data_generator_train,
validation_data=data_generator_val,
epochs=10, use_multiprocessing=True,
workers=num_cores,
verbose=1 
)
 model.save("model.h5" )
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
train_data = pd.read_csv("/kaggle/input/titanic/train.csv")
<feature_engineering>
class_names = ['Fish', 'Flower', 'Sugar', 'Gravel']
def get_threshold_for_recall(y_true, y_pred, class_i, recall_threshold=0.95, precision_threshold=0.94, plot=False):
precision, recall, thresholds = precision_recall_curve(y_true[:, class_i], y_pred[:, class_i])
 i = len(thresholds)- 1
best_recall_threshold = None
while best_recall_threshold is None:
next_threshold = thresholds[i]
next_recall = recall[i]
if next_recall >= recall_threshold:
best_recall_threshold = next_threshold
i -= 1
best_precision_threshold = [thres for prec, thres in zip(precision, thresholds)if prec >= precision_threshold][0]
if plot:
plt.figure(figsize=(10, 7))
plt.step(recall, precision, color='r', alpha=0.3, where='post')
 plt.fill_between(recall, precision, alpha=0.3, color='r')
 plt.axhline(y=precision[i + 1])
 recall_for_prec_thres = [rec for rec, thres in zip(recall, thresholds)
 if thres == best_precision_threshold][0]
plt.axvline(x=recall_for_prec_thres, color='g')
 plt.xlabel('Recall')
 plt.ylabel('Precision')
 plt.ylim([0.0, 1.05])
 plt.xlim([0.0, 1.0])
 plt.legend(['PR curve',
f'Precision {precision[i + 1]:.2f} corresponding to selected recall threshold',
f'Recall {recall_for_prec_thres:.2f} corresponding to selected precision threshold'])
 plt.title(f'Precision-Recall curve for Class {class_names[class_i]}')
 return best_recall_threshold, best_precision_threshold
y_pred = model.predict_generator(data_generator_val, workers=num_cores, verbose=1)
 y_true = data_generator_val.get_labels()
recall_thresholds = dict()
precision_thresholds = dict()
for i, class_name in tqdm(enumerate(class_names)) :
recall_thresholds[class_name], precision_thresholds[class_name] = get_threshold_for_recall(y_true, y_pred, i, plot=True )
train_data["Name"] = train_data["Name"].str.split(',' ).str[1]
train_data["Name"] = train_data["Name"].str.split('.' ).str[0]
train_data["Name"] = train_data["Name"].str.strip()<groupby>
data_generator_test = DataGenenerator(folder_imgs=test_imgs_folder, shuffle=False)
 y_pred_test = model.predict_generator(data_generator_test, workers=num_cores, verbose=1 )
x = train_data.groupby('Name' ).agg(['count'] ).index.get_level_values('Name')
 x<categorify>
image_labels_empty = set()
for i,(img, predictions)in enumerate(zip(os.listdir(test_imgs_folder), y_pred_test)) :
for class_i, class_name in enumerate(class_names):
if predictions[class_i] < recall_thresholds[class_name]:
image_labels_empty.add(f'{img}_{class_name}' )
train_data["Age"] = train_data.groupby("Name" ).transform(lambda x: x.fillna(x.mean())) ['Age']
train_data['Sex'].replace({'female':0,'male':1},inplace=True)
 train_data.head()<define_variables>
submission = pd.read_csv('.. /input/efficient-net-b4-unet-clouds/submission.csv')
 submission.head()
train_data_log = train_data.iloc[:,[False,False,True, False,True,True,True,True,False,True,False,False]]
normalized_data_train=(train_data_log-train_data_log.min())/(train_data_log.max() -train_data_log.min())
 train_labels_log = train_data.iloc[:,1]
normalized_data_train.head()<compute_test_metric>
predictions_nonempty = set(submission.loc[~submission['EncodedPixels'].isnull() , 'Image_Label'].values )
def log_odds(data,coefficients,intercept):

 return np.dot(data,coefficients)+ intercept
l_o= log_odds(normalized_data_train,initial_coefs, initial_intercept)
<compute_test_metric>
print(f'{len(image_labels_empty.intersection(predictions_nonempty)) } masks would be removed' )
def sigmoid(log_odds_vars):

 sigmoid_values = 1/(1+np.exp(-log_odds_vars))
return sigmoid_values
sigmoid_vals = sigmoid(l_o)
<compute_test_metric>
submission.loc[submission['Image_Label'].isin(image_labels_empty), 'EncodedPixels'] = np.nan
submission.to_csv('submission_segmentation_and_classifier.csv', index=None )
<compute_train_metric><EOS> 
FileLink(r'submission_segmentation_and_classifier.csv' )