Kaggle competition: Understanding Clouds from Satellite Images
<SOS> metric: Dice Kaggle data source: understanding-clouds-from-satellite-images<choose_model_class>
!pip install -U keras-applications
densenet = DenseNet121(
 weights='.. /input/densenet-keras/DenseNet-BC-121-32-no-top.h5',
include_top=False,
input_shape=(32,32,3)
 )<choose_model_class>
import keras_applications
import keras
import tensorflow as tf
model = Sequential()
model.add(densenet)
 model.add(layers.GlobalAveragePooling2D())
 model.add(layers.Dense(14, activation='softmax'))<train_model>
from keras_applications.resnext import ResNeXt101
model.compile(
 loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'] 
)
 checkpoint = ModelCheckpoint(
 'model.h5',
monitor='val_acc',
verbose=1,
save_best_only=True,
save_weights_only=False,
mode='auto' 
)
 history = model.fit(
 x=x_train,
y=y_train,
batch_size=64,
epochs=7,
callbacks=[checkpoint, f1_metrics],
validation_split=0.1
)<load_from_csv>
seed(10)
 set_random_seed(10)
 %matplotlib inline
model.load_weights('model.h5')
 y_test = model.predict(x_test)
 submission_df = pd.read_csv('.. /input/iwildcam-2019-fgvc6/sample_submission.csv')
 submission_df['Predicted'] = y_test.argmax(axis=1)
 print(submission_df.shape)
 submission_df.head()<save_to_csv>
test_imgs_folder = '.. /input/understanding_cloud_organization/test_images/'
train_imgs_folder = '.. /input/understanding_cloud_organization/train_images/'
num_cores = multiprocessing.cpu_count()
submission_df.to_csv('submission.csv',index=False )<load_from_csv>
train_df = pd.read_csv('.. /input/understanding_cloud_organization/train.csv')
 train_df.head()
np.random.seed(2018)
 train = pd.read_csv('.. /input/train.csv', usecols =['Survived','PassengerId','Name'])
 test = pd.read_csv('.. /input/test.csv', usecols =['PassengerId','Name'])
 test['Survived'] = np.nan
comb = pd.concat([ train, test ])
 comb.head()<string_transform>
train_df = train_df[~train_df['EncodedPixels'].isnull() ]
train_df['Image'] = train_df['Image_Label'].map(lambda x: x.split('_')[0])
 train_df['Class'] = train_df['Image_Label'].map(lambda x: x.split('_')[1])
 classes = train_df['Class'].unique()
train_df = train_df.groupby('Image')['Class'].agg(set ).reset_index()
for class_name in classes:
train_df[class_name] = train_df['Class'].map(lambda x: 1 if class_name in x else 0)
 train_df.head()
def clean_name(x):
x = x.replace(',',' ' ).replace('.',' ' ).replace('(',' ' ).replace(')',' ' ).replace('"',' ' ).replace('-',' ')
 return ' '.join([ w for w in x.split() if len(w)> 1] )<choose_model_class>
img_2_ohe_vector = {img:vec for img, vec in zip(train_df['Image'], train_df.iloc[:, 2:].values)}
count_vect = CountVectorizer(preprocessor=clean_name, min_df=2 )<train_model>
train_imgs, val_imgs = train_test_split(train_df['Image'].values,
test_size=0.1,
stratify=train_df['Class'].map(lambda x: str(sorted(list(x)))) ,
random_state=43 )
count_vect.fit(comb['Name'])
 for i,k in enumerate(count_vect.vocabulary_):
if i>5:
break
else:
print(k, count_vect.vocabulary_[k] )<categorify>
class DataGenenerator(Sequence):
def __init__(self, images_list=None, folder_imgs=train_imgs_folder,
batch_size=32, shuffle=True, augmentation=None,
resized_height=224, resized_width=224, num_channels=3):
self.batch_size = batch_size
self.shuffle = shuffle
self.augmentation = augmentation
if images_list is None:
self.images_list = os.listdir(folder_imgs)
 else:
self.images_list = deepcopy(images_list)
 self.folder_imgs = folder_imgs
self.len = len(self.images_list)// self.batch_size
self.resized_height = resized_height
self.resized_width = resized_width
self.num_channels = num_channels
self.num_classes = 4
self.is_test = not 'train' in folder_imgs
if not shuffle and not self.is_test:
self.labels = [img_2_ohe_vector[img] for img in self.images_list[:self.len*self.batch_size]]
def __len__(self):
return self.len
def on_epoch_start(self):
if self.shuffle:
random.shuffle(self.images_list)
 def __getitem__(self, idx):
current_batch = self.images_list[idx * self.batch_size:(idx + 1)* self.batch_size]
X = np.empty(( self.batch_size, self.resized_height, self.resized_width, self.num_channels))
y = np.empty(( self.batch_size, self.num_classes))
for i, image_name in enumerate(current_batch):
path = os.path.join(self.folder_imgs, image_name)
 img = cv2.resize(cv2.imread(path),(self.resized_height, self.resized_width)).astype(np.float32)
 if not self.augmentation is None:
augmented = self.augmentation(image=img)
 img = augmented['image']
X[i, :, :, :] = img/255.0
if not self.is_test:
y[i, :] = img_2_ohe_vector[image_name]
return X, y
def get_labels(self):
if self.shuffle:
images_current = self.images_list[:self.len*self.batch_size]
labels = [img_2_ohe_vector[img] for img in images_current]
else:
labels = self.labels
return np.array(labels )
v = count_vect.transform(comb['Name'])
 va = np.array(v.toarray())
 va.shape<filter>
albumentations_train = Compose([
VerticalFlip() , HorizontalFlip() , Rotate(limit=20), GridDistortion()
], p=1 )
pca = PCA(n_components = 6, random_state = 2018, whiten = True)
 va = pca.fit_transform(va)
 va_survived = va[comb.index[comb.loc[:,'Survived']==1],...]
va_perished = va[comb.index[comb.loc[:,'Survived']==0],...]
va_na = va[comb.index[comb.loc[:,'Survived']== np.nan],...]<prepare_output>
data_generator_train = DataGenenerator(train_imgs, augmentation=albumentations_train)
 data_generator_train_eval = DataGenenerator(train_imgs, shuffle=False)
 data_generator_val = DataGenenerator(val_imgs, shuffle=False )
feature_names = ['Nv' + str(i)for i in range(va.shape[1])]
NameVect = pd.DataFrame(data = va, index = comb.index, columns = feature_names)
 NameVect.head()<save_to_csv>
class PrAucCallback(Callback):
def __init__(self, data_generator, num_workers=num_cores,
early_stopping_patience=3,
plateau_patience=3, reduction_rate=0.5,
stage='train', checkpoints_path='checkpoints/'):
super(Callback, self ).__init__()
self.data_generator = data_generator
self.num_workers = num_workers
self.class_names = ['Fish', 'Flower', 'Sugar', 'Gravel']
self.history = [[] for _ in range(len(self.class_names)+ 1)]
self.early_stopping_patience = early_stopping_patience
self.plateau_patience = plateau_patience
self.reduction_rate = reduction_rate
self.stage = stage
self.best_pr_auc = -float('inf')
 if not os.path.exists(checkpoints_path):
os.makedirs(checkpoints_path)
 self.checkpoints_path = checkpoints_path
def compute_pr_auc(self, y_true, y_pred):
pr_auc_mean = 0
print(f"
{'
")
 for class_i in range(len(self.class_names)) :
precision, recall, _ = precision_recall_curve(y_true[:, class_i], y_pred[:, class_i])
 pr_auc = auc(recall, precision)
 pr_auc_mean += pr_auc/len(self.class_names)
 print(f"PR AUC {self.class_names[class_i]}, {self.stage}: {pr_auc:.3f}
")
 self.history[class_i].append(pr_auc)
 print(f"
{'
PR AUC mean, {self.stage}: {pr_auc_mean:.3f}
{'
")
 self.history[-1].append(pr_auc_mean)
 return pr_auc_mean
def is_patience_lost(self, patience):
if len(self.history[-1])> patience:
best_performance = max(self.history[-1][-(patience + 1):-1])
 return best_performance == self.history[-1][-(patience + 1)] and best_performance >= self.history[-1][-1]
def early_stopping_check(self, pr_auc_mean):
if self.is_patience_lost(self.early_stopping_patience):
self.model.stop_training = True
def model_checkpoint(self, pr_auc_mean, epoch):
if pr_auc_mean > self.best_pr_auc:
for checkpoint in glob.glob(os.path.join(self.checkpoints_path, 'classifier_epoch_*')) :
os.remove(checkpoint)
 self.best_pr_auc = pr_auc_mean
self.model.save(os.path.join(self.checkpoints_path, f'classifier_epoch_{epoch}_val_pr_auc_{pr_auc_mean}.h5'))
print(f"
{'
Saved new checkpoint
{'
")
 def reduce_lr_on_plateau(self):
if self.is_patience_lost(self.plateau_patience):
new_lr = float(keras.backend.get_value(self.model.optimizer.lr)) * self.reduction_rate
keras.backend.set_value(self.model.optimizer.lr, new_lr)
 print(f"
{'
Reduced learning rate to {new_lr}.
 {'
")
 def on_epoch_end(self, epoch, logs={}):
y_pred = self.model.predict_generator(self.data_generator, workers=self.num_workers)
 y_true = self.data_generator.get_labels()
pr_auc_mean = self.compute_pr_auc(y_true, y_pred)
 if self.stage == 'val':
self.early_stopping_check(pr_auc_mean)
 self.model_checkpoint(pr_auc_mean, epoch)
 self.reduce_lr_on_plateau()
def get_pr_auc_history(self):
return self.history
comb2 = pd.concat([comb[['Survived']], NameVect],axis =1)
 comb2.head()
comb2.to_csv('name_only_df.csv',index=False )<filter>
train_metric_callback = PrAucCallback(data_generator_train_eval)
 val_callback = PrAucCallback(data_generator_val, stage='val' )
df_train = comb2.loc[comb2['Survived'].isin([np.nan])== False]
df_test = comb2.loc[comb2['Survived'].isin([np.nan])== True]
print(df_train.shape)
 df_train.head()<import_modules>
def get_model() :
base_model = model = ResNeXt101 (..., backend=tf.keras.backend, layers=tf.keras.layers, weights = 'imagenet', models=tf.keras.models, utils=tf.keras.utils)
 x = base_model.output
y_pred = Dense(4, activation='sigmoid' )(x)
 return Model(inputs=base_model.input, outputs=y_pred)
 model = get_model()
from sklearn.model_selection import GridSearchCV<import_modules>
for base_layer in model.layers[:-1]:
base_layer.trainable = False
model.compile(optimizer=Adam(lr=0.0001), loss='categorical_crossentropy')
 history_0 = model.fit_generator(generator=data_generator_train,
validation_data=data_generator_val,
epochs=15,
callbacks=[train_metric_callback, val_callback],
workers=num_cores,
verbose=1
)
from sklearn.model_selection import GridSearchCV<train_on_grid>
for base_layer in model.layers[:-1]:
base_layer.trainable = True
model.compile(optimizer=Adam(lr=0.0001), loss='binary_crossentropy')
 history_1 = model.fit_generator(generator=data_generator_train,
validation_data=data_generator_val,
epochs=20,
callbacks=[train_metric_callback, val_callback],
workers=num_cores,
verbose=1,
initial_epoch=1
)
knclass = KNeighborsClassifier(n_neighbors=11, metric = 'manhattan')
 param_grid =({'n_neighbors':[3,4,5,6,7,8,9,11,12,13],'metric':['manhattan','minkowski'],'p':[1,2]})
 grs = GridSearchCV(knclass, param_grid, cv = 28, n_jobs=1, return_train_score = True, iid = False)
 grs.fit(np.array(df_train[feature_names]), np.array(df_train['Survived']))<compute_test_metric>
model = load_model('.. /input/clouds-classifier-files/classifier_epoch_45_val_pr_auc_0.8344173287108075.h5' )
print("Best parameters " + str(grs.best_params_))
gpd = pd.DataFrame(grs.cv_results_)
 print("Estimated accuracy of this model for unseen data:{0:1.4f}".format(gpd['mean_test_score'][grs.best_index_]))<save_to_csv>
Image(".. /input/clouds-classifier-files/loss_hist.png" )
pred_knn = grs.predict(np.array(df_test[feature_names]))
sub = pd.DataFrame({'PassengerId':test['PassengerId'],'Survived':pred_knn})
 sub.to_csv('name_only_knn.csv', index = False, float_format='%1d')
 sub.head()<set_options>
Image(".. /input/clouds-classifier-files/pr_auc_hist.png" )
warnings.simplefilter("ignore" )<set_options>
Image(".. /input/clouds-classifier-files/training_hist_no_aug.png" )
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns; sns.set()<load_from_csv>
class_names = ['Fish', 'Flower', 'Sugar', 'Gravel']
def get_threshold_for_recall(y_true, y_pred, class_i, recall_threshold=0.94, precision_threshold=0.95, plot=False):
precision, recall, thresholds = precision_recall_curve(y_true[:, class_i], y_pred[:, class_i])
 i = len(thresholds)- 1
best_recall_threshold = None
while best_recall_threshold is None:
next_threshold = thresholds[i]
next_recall = recall[i]
if next_recall >= recall_threshold:
best_recall_threshold = next_threshold
i -= 1
best_precision_threshold = [thres for prec, thres in zip(precision, thresholds)if prec >= precision_threshold][0]
if plot:
plt.figure(figsize=(10, 7))
plt.step(recall, precision, color='r', alpha=0.3, where='post')
 plt.fill_between(recall, precision, alpha=0.3, color='r')
 plt.axhline(y=precision[i + 1])
 recall_for_prec_thres = [rec for rec, thres in zip(recall, thresholds)
 if thres == best_precision_threshold][0]
plt.axvline(x=recall_for_prec_thres, color='g')
 plt.xlabel('Recall')
 plt.ylabel('Precision')
 plt.ylim([0.0, 1.05])
 plt.xlim([0.0, 1.0])
 plt.legend(['PR curve',
f'Precision {precision[i + 1]:.2f} corresponding to selected recall threshold',
f'Recall {recall_for_prec_thres:.2f} corresponding to selected precision threshold'])
 plt.title(f'Precision-Recall curve for Class {class_names[class_i]}')
 return best_recall_threshold, best_precision_threshold
y_pred = model.predict_generator(data_generator_val, workers=num_cores)
 y_true = data_generator_val.get_labels()
recall_thresholds = dict()
precision_thresholds = dict()
for i, class_name in tqdm(enumerate(class_names)) :
recall_thresholds[class_name], precision_thresholds[class_name] = get_threshold_for_recall(y_true, y_pred, i, plot=True )
train_df = pd.read_csv('.. /input/train.csv')
 test_df = pd.read_csv('.. /input/test.csv')
 sub = pd.read_csv('.. /input/gender_submission.csv' )<count_missing_values>
data_generator_test = DataGenenerator(folder_imgs=test_imgs_folder, shuffle=False)
 y_pred_test = model.predict_generator(data_generator_test, workers=num_cores )
train_df.isnull().sum()<count_missing_values>
image_labels_empty = set()
for i,(img, predictions)in enumerate(zip(os.listdir(test_imgs_folder), y_pred_test)) :
for class_i, class_name in enumerate(class_names):
if predictions[class_i] < recall_thresholds[class_name]:
image_labels_empty.add(f'{img}_{class_name}' )
test_df.isnull().sum()<categorify>
images_labels_full = set()
for i,(img, predictions)in enumerate(zip(os.listdir(test_imgs_folder), y_pred_test)) :
for class_i, class_name in enumerate(class_names):
if predictions[class_i] > precision_thresholds[class_name]:
images_labels_full.add(f'{img}_{class_name}' )
imputer = SimpleImputer(np.nan, "mean")
 train_df['Age'] = imputer.fit_transform(np.array(train_df['Age'] ).reshape(891, 1))
train_df.Embarked.fillna(method='ffill', inplace=True)
 train_df.drop(['PassengerId', 'Name', 'Cabin'], axis=1, inplace=True)
 test_df['Age'] = imputer.fit_transform(np.array(test_df['Age'] ).reshape(418, 1))
test_df.Embarked.fillna(method='ffill', inplace=True)
 test_df.Fare.fillna(method='ffill', inplace=True)
 test_df.drop(['PassengerId', 'Name', 'Cabin'], axis=1, inplace=True )<drop_column>
submission = pd.read_csv('.. /input/cloudefficientnetb2/submission.csv')
 submission.head()
<categorify>
image_labels_andrew_nonempty = submission.loc[~submission['EncodedPixels'].isnull() , 'Image_Label'].values
image_labels_andrew_empty = submission.loc[submission['EncodedPixels'].isnull() , 'Image_Label'].values
le = LabelEncoder()
train_df[objects_cols] = train_df[objects_cols].apply(le.fit_transform)
 test_df[objects_cols] = test_df[objects_cols].apply(le.fit_transform)
 train_df[objects_cols].head()<import_modules>
images_to_refill = images_labels_full.intersection(set(image_labels_andrew_empty))
submission_no_minsize = pd.read_csv('.. /input/clouds-classifier-files/submission_no_minsize.csv')
 masks_no_minsize_refillers = submission_no_minsize.loc[submission_no_minsize['Image_Label'].isin(images_to_refill)&
~submission_no_minsize['EncodedPixels'].isnull() ]
print(f'{len(masks_no_minsize_refillers)} masks would be added.' )
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.preprocessing import StandardScaler
from xgboost import XGBClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.svm import SVC
from sklearn.metrics import accuracy_score<prepare_x_and_y>
print(f'{len(image_labels_empty.intersection(set(image_labels_andrew_nonempty)))} masks would be removed' )
<normalization><EOS> 
submission.loc[submission['Image_Label'].isin(image_labels_empty), 'EncodedPixels'] = np.nan
img_labels_to_refill = set(masks_no_minsize_refillers['Image_Label'].values)
 submission = submission[~submission['Image_Label'].isin(img_labels_to_refill)]
submission = pd.concat(( submission, masks_no_minsize_refillers))
submission.to_csv('submission.csv', index=None )