Kaggle competition: Understanding Clouds from Satellite Images
<SOS> metric: Dice Kaggle data source: understanding-clouds-from-satellite-images<compute_train_metric>
!pip install tta-wrapper --quiet
seed = 0
seed_everything(seed)
 warnings.filterwarnings("ignore" )
best_coef = best_coefs[0]
best_int = best_coefs[1]
v = sigmoid(log_odds(normalized_data_train,best_coef,best_int)) 
<define_variables>
train = pd.read_csv('.. /input/understanding_cloud_organization/train.csv')
 submission = pd.read_csv('.. /input/understanding_cloud_organization/sample_submission.csv')
 train['image'] = train['Image_Label'].apply(lambda x: x.split('_')[0])
 train['label'] = train['Image_Label'].apply(lambda x: x.split('_')[1])
 submission['image'] = submission['Image_Label'].apply(lambda x: x.split('_')[0])
 test = pd.DataFrame(submission['image'].unique() , columns=['image'])
 train_df = pd.pivot_table(train, index=['image'], values=['EncodedPixels'], columns=['label'], aggfunc=np.min ).reset_index()
train_df.columns = ['image', 'Fish_mask', 'Flower_mask', 'Gravel_mask', 'Sugar_mask']
print('Compete set samples:', len(train_df))
print('Test samples:', len(submission))
display(train.head() )
def find_threshold(sigmoid_vals):

 predictions = []
vals = []
accuracies = []
for num in range(1000):
vals.append(num/1000)
 accuracy = 0
for i in v:
if i > num/1000:
predictions.append(1)
 else:
predictions.append(0)
 for j in range(len(predictions)) :
if predictions[j] == train_labels_log[j]:
accuracy += 1
accuracies.append(accuracy/len(predictions))
accuracy = 0
predictions = []
indx = accuracies.index(max(accuracies))
print("Best accuracy on training set:")
 print(max(accuracies))
best_threshold = vals[indx]
return best_threshold
best_thresh = find_threshold(v)
 print(best_thresh )<categorify>
X_train, X_val = train_test_split(train_df, test_size=0.2, random_state=seed)
 X_train['set'] = 'train'
X_val['set'] = 'validation'
test['set'] = 'test'
print('Train samples: ', len(X_train))
print('Validation samples: ', len(X_val))
def calculate_precision(sigmoid_vals, threshold, labels):
"Precision is True Positives /(True Positives + False Positives)"
predictions = []
true_positives = 0
false_positives = 0
for i in sigmoid_vals:
if i > threshold:
predictions.append(1)
 else:
predictions.append(0)
 for i in range(len(labels)) :
if labels[i] == 1 and labels[i] == predictions[i]:
true_positives += 1
elif labels[i] == 0 and labels[i] != predictions[i]:
false_positives += 1
return true_positives/(true_positives + false_positives)
 print("Precision:")
 print(calculate_precision(v, best_thresh, train_labels_log)) 
<define_variables>
BACKBONE = 'resnet18'
BATCH_SIZE = 32
EPOCHS = 12
LEARNING_RATE = 3e-4
HEIGHT = 384
WIDTH = 480
CHANNELS = 3
N_CLASSES = 4
ES_PATIENCE = 5
RLROP_PATIENCE = 3
DECAY_DROP = 0.5
model_path = 'uNet_%s_%sx%s.h5' %(BACKBONE, HEIGHT, WIDTH )
def calculate_recall(sigmoid_vals, threshold, labels):
"Precision is True Positives /(True Positives + False Negatives)"
predictions = []
true_positives = 0
false_negatives = 0
for i in sigmoid_vals:
if i > threshold:
predictions.append(1)
 else:
predictions.append(0)
 for i in range(len(labels)) :
if labels[i] == 1 and labels[i] == predictions[i]:
true_positives += 1
elif labels[i] == 1 and labels[i] != predictions[i]:
false_negatives += 1
return true_positives/(true_positives + false_negatives)
 print("Recall")
 print(calculate_recall(v, best_thresh, train_labels_log)) 
<feature_engineering>
preprocessing = sm.get_preprocessing(BACKBONE)
 augmentation = albu.Compose([albu.HorizontalFlip(p=0.5),
albu.VerticalFlip(p=0.5),
albu.ShiftScaleRotate(rotate_limit=30, shift_limit=0.1, p=0.5)
 ] )
test_data = pd.read_csv("/kaggle/input/titanic/test.csv")
 test_data["Name"] = test_data["Name"].str.split(',' ).str[1]
test_data["Name"] = test_data["Name"].str.split('.' ).str[0]
test_data["Name"] = test_data["Name"].str.strip()
test_data['Sex'].replace({'female':0,'male':1},inplace=True)
 x = test_data.groupby('Name' ).agg(['count'] ).index.get_level_values('Name')
 test_data["Age"] = test_data.groupby("Name" ).transform(lambda x: x.fillna(x.mean())) ['Age']
test_data_log = test_data.iloc[:,[False,True,False,True,True,True,True,False,True,False,False]]
normalized_data_test=(test_data_log-test_data_log.min())/(test_data_log.max() -test_data_log.min())
<compute_test_metric>
train_generator = DataGenerator(
 directory=train_images_dest_path,
dataframe=X_train,
target_df=train,
batch_size=BATCH_SIZE,
target_size=(HEIGHT, WIDTH),
n_channels=CHANNELS,
n_classes=N_CLASSES,
preprocessing=preprocessing,
augmentation=augmentation,
seed=seed)
 valid_generator = DataGenerator(
 directory=validation_images_dest_path,
dataframe=X_val,
target_df=train,
batch_size=BATCH_SIZE,
target_size=(HEIGHT, WIDTH),
n_channels=CHANNELS,
n_classes=N_CLASSES,
preprocessing=preprocessing,
seed=seed )
pred_test = sigmoid(log_odds(normalized_data_test,best_coef,best_int))<define_variables>
model = sm.Unet(backbone_name=BACKBONE,
encoder_weights='imagenet',
classes=N_CLASSES,
activation='sigmoid',
input_shape=(HEIGHT, WIDTH, CHANNELS))
checkpoint = ModelCheckpoint(model_path, monitor='val_loss', mode='min', save_best_only=True, save_weights_only=True)
 es = EarlyStopping(monitor='val_loss', mode='min', patience=ES_PATIENCE, restore_best_weights=True, verbose=1)
 rlrop = ReduceLROnPlateau(monitor='val_loss', mode='min', patience=RLROP_PATIENCE, factor=DECAY_DROP, min_lr=1e-6, verbose=1)
 metric_list = [dice_coef, sm.metrics.iou_score]
callback_list = [checkpoint, es, rlrop]
optimizer = RAdam(learning_rate=LEARNING_RATE, warmup_proportion=0.1)
 model.compile(optimizer=optimizer, loss=sm.losses.bce_dice_loss, metrics=metric_list)
 model.summary()
classifier = []
for i in range(len(pred_test)) :
if pred_test[i] > best_thresh:
classifier.append(1)
 else:
classifier.append(0 )<save_to_csv>
STEP_SIZE_TRAIN = len(X_train)//BATCH_SIZE
STEP_SIZE_VALID = len(X_val)//BATCH_SIZE
history = model.fit_generator(generator=train_generator,
steps_per_epoch=STEP_SIZE_TRAIN,
validation_data=valid_generator,
validation_steps=STEP_SIZE_VALID,
callbacks=callback_list,
epochs=EPOCHS,
verbose=2 ).history
data = {'PassengerId': test_data["PassengerId"].values, 'Survived':classifier}
df_submission = pd.DataFrame(data)
 df_submission.to_csv("submission_log_regression2.csv",index=False)
<set_options>
model = load_model('.. /input/cloud-seg-resnet18-trainedlonger/resnet18_trained_longer.h5', custom_objects={'RAdam':RAdam, 'binary_crossentropy_plus_dice_loss':sm.losses.bce_dice_loss, 'dice_coef':dice_coef, 'iou_score':sm.metrics.iou_score, 'f1-score':sm.metrics.f1_score} )
assert sys.version_info >=(3, 5)
 assert sklearn.__version__ >= "0.20"
%matplotlib inline
warnings.filterwarnings(action="ignore" )<load_from_csv>
class_names = ['Fish ', 'Flower', 'Gravel', 'Sugar ']
best_tresholds = [.5,.5,.5,.35]
best_masks = [25000, 20000, 22500, 15000]
for index, name in enumerate(class_names):
print('%s treshold=%.2f mask size=%d' %(name, best_tresholds[index], best_masks[index]))
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )<create_dataframe>
train_metrics = get_metrics(model, train, X_train, train_images_dest_path, best_tresholds, best_masks, seed=seed, preprocessing=preprocessing, set_name='Train')
 display(train_metrics)
 validation_metrics = get_metrics(model, train, X_val, validation_images_dest_path, best_tresholds, best_masks, seed=seed, preprocessing=preprocessing, set_name='Validation')
 display(validation_metrics )
passengerID = test['PassengerId'].copy()<concatenate>
model = tta_segmentation(model, h_flip=True, v_flip=True, h_shift=(-10, 10), v_shift=(-10, 10), merge='mean' )
train["split"] = "train"
test["split"] = "test"
data = pd.concat([train , test], ignore_index=True)
 data.set_index("PassengerId")
 data["Survived"].fillna(0.0,inplace=True)
 print(train.shape, test.shape, data.shape )<count_unique_values>
test_df = []
for i in range(0, test.shape[0], 300):
batch_idx = list(range(i, min(test.shape[0], i + 300)))
 batch_set = test[batch_idx[0]: batch_idx[-1]+1]
test_generator = DataGenerator(
 directory=test_images_dest_path,
dataframe=batch_set,
target_df=submission,
batch_size=1,
target_size=(HEIGHT, WIDTH),
n_channels=CHANNELS,
n_classes=N_CLASSES,
preprocessing=preprocessing,
seed=seed,
mode='predict',
shuffle=False)
 preds = model.predict_generator(test_generator)
 for index, b in enumerate(batch_idx):
filename = test['image'].iloc[b]
image_df = submission[submission['image'] == filename].copy()
pred_masks = preds[index, ].round().astype(int)
 pred_rles = build_rles(pred_masks, reshape=(350, 525))
image_df['EncodedPixels'] = pred_rles
pred_masks_post = preds[index, ].astype('float32')
 for class_index in range(N_CLASSES):
pred_mask = pred_masks_post[...,class_index]
pred_mask = post_process(pred_mask, threshold=best_tresholds[class_index], min_size=best_masks[class_index])
 pred_masks_post[...,class_index] = pred_mask
pred_rles_post = build_rles(pred_masks_post, reshape=(350, 525))
image_df['EncodedPixels_post'] = pred_rles_post
test_df.append(image_df)
 sub_df = pd.concat(test_df )
data.apply(lambda x: len(x.unique()))<feature_engineering>
images_to_inspect = np.random.choice(X_val['image'].unique() , 3, replace=False)
 inspect_set = train[train['image'].isin(images_to_inspect)].copy()
inspect_set_temp = []
inspect_generator = DataGenerator(
 directory=validation_images_dest_path,
dataframe=inspect_set,
target_df=train,
batch_size=1,
target_size=(HEIGHT, WIDTH),
n_channels=CHANNELS,
n_classes=N_CLASSES,
preprocessing=preprocessing,
seed=seed,
mode='fit',
shuffle=False)
 preds = model.predict_generator(inspect_generator)
 for index, b in enumerate(range(len(preds))):
filename = inspect_set['image'].iloc[b]
image_df = inspect_set[inspect_set['image'] == filename].copy()
pred_masks = preds[index, ].round().astype(int)
 pred_rles = build_rles(pred_masks, reshape=(350, 525))
image_df['EncodedPixels_pred'] = pred_rles
pred_masks_post = preds[index, ].astype('float32')
 for class_index in range(N_CLASSES):
pred_mask = pred_masks_post[...,class_index]
pred_mask = post_process(pred_mask, threshold=best_tresholds[class_index], min_size=best_masks[class_index])
 pred_masks_post[...,class_index] = pred_mask
pred_rles_post = build_rles(pred_masks_post, reshape=(350, 525))
image_df['EncodedPixels_pred_post'] = pred_rles_post
inspect_set_temp.append(image_df)
 inspect_set = pd.concat(inspect_set_temp)
 inspect_predictions(inspect_set, images_to_inspect, validation_images_dest_path, pred_col='EncodedPixels_pred' )
data['FamilySize'] = data['SibSp'] + data['Parch'] + 1
data['IsAlone'] = 0
data.loc[data['FamilySize'] == 1, 'IsAlone'] = 1<count_values>
inspect_predictions(inspect_set, images_to_inspect, validation_images_dest_path, pred_col='EncodedPixels_pred_post' )
data.groupby('Fare')['Fare'].unique().value_counts()<feature_engineering>
images_to_inspect_test = np.random.choice(sub_df['image'].unique() , 4, replace=False)
 inspect_predictions(sub_df, images_to_inspect_test, test_images_dest_path )
data['Fare'].fillna(data['Fare'].median() , inplace=True)
 data.loc[data['Fare'] <= 7.91, 'Fare'] = 0
data.loc[(data['Fare'] > 7.91)&(data['Fare'] <= 14.454), 'Fare'] = 1
data.loc[(data['Fare'] > 14.454)&(data['Fare'] <= 31), 'Fare'] = 2
data.loc[ data['Fare'] > 31, 'Fare'] = 3
data['Fare'] = data['Fare'].astype(int )<feature_engineering>
inspect_predictions(sub_df, images_to_inspect_test, test_images_dest_path, label_col='EncodedPixels_post' )
data['Has_Cabin'] = data["Cabin"].apply(lambda x: 0 if type(x)== float else 1 )<data_type_conversions>
submission_df = sub_df[['Image_Label' ,'EncodedPixels']]
submission_df.to_csv('submission.csv', index=False)
 display(submission_df.head() )
avg = data['Age'].mean()
std = data['Age'].std()
null_count = data['Age'].isnull().sum()
random_list = np.random.randint(avg - std, avg + std, size = null_count)
 data['Age'][np.isnan(data['Age'])] = random_list
data['Age'] = data['Age'].astype(int )<feature_engineering>
submission_df_post = sub_df[['Image_Label' ,'EncodedPixels_post']]
submission_df_post.columns = ['Image_Label' ,'EncodedPixels']
submission_df_post.to_csv('submission_post.csv', index=False)
 display(submission_df_post.head() )
<data_type_conversions><EOS> 
if os.path.exists(train_images_dest_path):
shutil.rmtree(train_images_dest_path)
 if os.path.exists(validation_images_dest_path):
shutil.rmtree(validation_images_dest_path)
 if os.path.exists(test_images_dest_path):
shutil.rmtree(test_images_dest_path )