Kaggle competition: Understanding Clouds from Satellite Images
for max_depth, min_child_weight in param_combinations:
score_folds = []
kf = KFold(n_splits=4, shuffle=True, random_state=123456)
 for tr_idx, va_idx in kf.split(train_x):
tr_x, va_x = train_x.iloc[tr_idx], train_x.iloc[va_idx]
tr_y, va_y = train_y.iloc[tr_idx], train_y.iloc[va_idx]
model = XGBClassifier(n_estimators=20, random_state=71, max_depth=max_depth, min_child_weight=min_child_weight)
 model.fit(tr_x, tr_y)
 va_pred = model.predict_proba(va_x)[:, 1]
logloss = log_loss(va_y, va_pred)
 score_folds.append(logloss)
 score_mean = np.mean(score_folds)
 params.append(( max_depth, min_child_weight))
scores.append(score_mean )<find_best_params>
!pip install catalyst
!pip install pretrainedmodels
!pip install git+https://github.com/qubvel/segmentation_models.pytorch
!pip install pytorch_toolbelt
!pip install torchvision==0.4
!pip install -v --no-cache-dir --global-option="--cpp_ext" --global-option="--cuda_ext".. /input/apex-325f5a0/apex-master/
best_idx = np.argsort(scores)[0]
best_param = params[best_idx]
print(f'max_depth: {best_param[0]}, min_child_weight: {best_param[1]}' )<import_modules>
train_on_gpu = True
%matplotlib inline
device=torch.device('cuda' )
from sklearn.preprocessing import OneHotEncoder<drop_column>
path = '.. /input/understanding_cloud_organization'
img_paths = '.. /input/understanding-clouds-resized'
os.listdir(path )
train_x2 = train.drop(['Survived'], axis=1)
 test_x2 = test.copy()<drop_column>
train = pd.read_csv(f'{path}/train.csv')
 sub = pd.read_csv(f'{path}/sample_submission.csv' )
train_x2 = train_x2.drop(['PassengerId'], axis=1)
 test_x2 = test_x2.drop(['PassengerId'], axis=1 )<drop_column>
n_train = len(os.listdir(f'{img_paths}/train_images_525/train_images_525'))
n_test = len(os.listdir(f'{img_paths}/test_images_525/test_images_525'))
print(f'There are {n_train} images in train dataset')
 print(f'There are {n_test} images in test dataset' )
train_x2 = train_x2.drop(['Name', 'Ticket', 'Cabin'], axis=1)
 test_x2 = test_x2.drop(['Name', 'Ticket', 'Cabin'], axis=1 )<categorify>
train['Image_Label'].apply(lambda x: x.split('_')[1] ).value_counts()
cat_cols = ['Sex', 'Embarked', 'Pclass']
ohe = OneHotEncoder(categories='auto', sparse=False)
 ohe.fit(train_x2[cat_cols].fillna('NA'))<define_variables>
train.loc[train['EncodedPixels'].isnull() == False, 'Image_Label'].apply(lambda x: x.split('_')[1] ).value_counts()
ohe_columns = []
for i, c in enumerate(cat_cols):
ohe_columns += [f'{c}_{v}' for v in ohe.categories_[i]]<create_dataframe>
train.loc[train['EncodedPixels'].isnull() == False, 'Image_Label'].apply(lambda x: x.split('_')[0] ).value_counts().value_counts()
ohe_train_x2 = pd.DataFrame(ohe.transform(train_x2[cat_cols].fillna('NA')) , columns=ohe_columns)
 ohe_test_x2 = pd.DataFrame(ohe.transform(test_x2[cat_cols].fillna('NA')) , columns=ohe_columns )<drop_column>
train['label'] = train['Image_Label'].apply(lambda x: x.split('_')[1])
 train['im_id'] = train['Image_Label'].apply(lambda x: x.split('_')[0])
 sub['label'] = sub['Image_Label'].apply(lambda x: x.split('_')[1])
 sub['im_id'] = sub['Image_Label'].apply(lambda x: x.split('_')[0] )
train_x2 = train_x2.drop(cat_cols, axis=1)
 test_x2 = test_x2.drop(cat_cols, axis=1 )<concatenate>
id_mask_count = train.loc[train['EncodedPixels'].isnull() == False, 'Image_Label'].apply(lambda x: x.split('_')[0] ).value_counts().\
reset_index().rename(columns={'index': 'img_id', 'Image_Label': 'count'})
 train_ids, valid_ids = train_test_split(id_mask_count['img_id'].values, random_state=42, stratify=id_mask_count['count'], test_size=0.1)
 test_ids = sub['Image_Label'].apply(lambda x: x.split('_')[0] ).drop_duplicates().values
train_x2 = pd.concat([train_x2, ohe_train_x2], axis=1)
 test_x2 = pd.concat([test_x2, ohe_test_x2], axis=1 )<data_type_conversions>
image_name = '8242ba0.jpg'
image = get_img(image_name)
 mask = make_mask(train, image_name )
num_cols = ['Age', 'SibSp', 'Parch', 'Fare']
for col in num_cols:
train_x2[col].fillna(train_x2[col].mean() , inplace=True)
 test_x2[col].fillna(train_x2[col].mean() , inplace=True )<feature_engineering>
class CloudDataset(Dataset):
def __init__(self, df: pd.DataFrame = None, datatype: str = 'train', img_ids: np.array = None,
transforms = albu.Compose([albu.HorizontalFlip() ,AT.ToTensor() ]),
preprocessing=None):
self.df = df
if datatype != 'test':
self.data_folder = f"{img_paths}/train_images_525/train_images_525"
else:
self.data_folder = f"{img_paths}/test_images_525/test_images_525"
self.img_ids = img_ids
self.transforms = transforms
self.preprocessing = preprocessing
def __getitem__(self, idx):
image_name = self.img_ids[idx]
mask = make_mask(self.df, image_name)
 image_path = os.path.join(self.data_folder, image_name)
 img = cv2.imread(image_path)
 img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
 augmented = self.transforms(image=img, mask=mask)
 img = augmented['image']
mask = augmented['mask']
if self.preprocessing:
preprocessed = self.preprocessing(image=img, mask=mask)
 img = preprocessed['image']
mask = preprocessed['mask']
return img, mask
def __len__(self):
return len(self.img_ids )
train_x2['Fare'] = np.log1p(train_x2['Fare'])
 test_x2['Fare'] = np.log1p(test_x2['Fare'] )<import_modules>
ENCODER = 'resnet18'
ENCODER_WEIGHTS = 'imagenet'
ACTIVATION = None
model = smp.Unet(
 encoder_name=ENCODER,
encoder_weights=ENCODER_WEIGHTS,
classes=4,
activation=ACTIVATION, 
)
 preprocessing_fn = smp.encoders.get_preprocessing_fn(ENCODER, ENCODER_WEIGHTS )
from sklearn.linear_model import LogisticRegression<train_model>
num_workers = 4
bs = 32
train_dataset = CloudDataset(df=train, datatype='train', img_ids=train_ids, transforms = get_training_augmentation() , preprocessing=get_preprocessing(preprocessing_fn))
valid_dataset = CloudDataset(df=train, datatype='valid', img_ids=valid_ids, transforms = get_validation_augmentation() , preprocessing=get_preprocessing(preprocessing_fn))
train_loader = DataLoader(train_dataset, batch_size=bs, shuffle=True, num_workers=num_workers)
 valid_loader = DataLoader(valid_dataset, batch_size=bs, shuffle=False, num_workers=num_workers)
 loaders = {
"train": train_loader,
"valid": valid_loader
}
model_xgb = XGBClassifier(n_estimators=20, random_state=71)
 model_xgb.fit(train_x, train_y)
 pred_xgb = model_xgb.predict_proba(test_x)[:, 1]<train_model>
def dice_loss(input, target):
input = torch.sigmoid(input)
 smooth = 1.0
iflat = input.view(-1)
 tflat = target.view(-1)
 intersection =(iflat * tflat ).sum()
return(( 2.0 * intersection + smooth)/(iflat.sum() + tflat.sum() + smooth))
class FocalLoss(nn.Module):
def __init__(self, gamma):
super().__init__()
self.gamma = gamma
def forward(self, input, target):
if not(target.size() == input.size()):
raise ValueError("Target size({})must be the same as input size({})" 
.format(target.size() , input.size()))
max_val =(-input ).clamp(min=0)
 loss = input - input * target + max_val + \ 
(( -max_val ).exp() +(-input - max_val ).exp() ).log()
invprobs = F.logsigmoid(-input *(target * 2.0 - 1.0))
loss =(invprobs * self.gamma ).exp() * loss
return loss.mean()
class MixedLoss(nn.Module):
def __init__(self, alpha, gamma):
super().__init__()
self.alpha = alpha
self.focal = FocalLoss(gamma)
 def forward(self, input, target):
loss = self.alpha*self.focal(input, target)- torch.log(dice_loss(input, target))
return loss.mean()
model_lr = LogisticRegression(solver='lbfgs', max_iter=300)
 model_lr.fit(train_x2, train_y)
 pred_lr = model_lr.predict_proba(test_x2)[:, 1]<define_variables>
num_epochs = 25
logdir = "./logs/segmentation_unet"
optimizer = torch.optim.Adam([
{'params': model.decoder.parameters() , 'lr': 1e-2},
{'params': model.encoder.parameters() , 'lr': 1e-3},
])
 opt_level = 'O1'
model.cuda()
model, optimizer = amp.initialize(model, optimizer, opt_level=opt_level)
 scheduler = ReduceLROnPlateau(optimizer, factor=0.5, patience=2)
 criterion = smp.utils.losses.BCEDiceLoss(eps=1.)
runner = SupervisedRunner()
pred = pred_xgb * 0.8 + pred_lr * 0.2
pred_label = np.where(pred > 0.5, 1, 0 )<save_to_csv>
runner.train(
 model=model,
criterion=criterion,
optimizer=optimizer,
scheduler=scheduler,
loaders=loaders,
callbacks=[DiceCallback() , EarlyStoppingCallback(patience=5, min_delta=0.001)],
logdir=logdir,
num_epochs=num_epochs,
verbose=True
)
submission = pd.DataFrame({'passengerId': test['PassengerId'], 'Survived': pred_label})
 submission.to_csv('submission_ensemble.csv', index=False )<set_options>
encoded_pixels = []
loaders = {"infer": valid_loader}
runner.infer(
 model=model,
loaders=loaders,
callbacks=[
CheckpointCallback(
 resume=f"{logdir}/checkpoints/best.pth"),
InferCallback()
], 
)
 valid_masks = []
probabilities = np.zeros(( 2220, 350, 525), dtype = np.float32)
 for i,(batch, output)in enumerate(tqdm.tqdm(zip(
 valid_dataset, runner.callbacks[0].predictions["logits"]))):
image, mask = batch
for m in mask:
if m.shape !=(350, 525):
m = cv2.resize(m, dsize=(525, 350), interpolation=cv2.INTER_LINEAR)
 valid_masks.append(m)
 for j, probability in enumerate(output):
if probability.shape !=(350, 525):
probability = cv2.resize(probability, dsize=(525, 350), interpolation=cv2.INTER_LINEAR)
 probabilities[i * 4 + j, :, :] = probability
%matplotlib inline 
<load_from_csv>
torch.cuda.empty_cache()
gc.collect()
train_df = pd.read_csv('.. /input/train.csv')
 test_df = pd.read_csv('.. /input/test.csv')
 combine = [train_df, test_df]<sort_values>
class_params = {}
for class_id in range(4):
print(class_id)
 attempts = []
for t in range(0, 100, 5):
t /= 100
for ms in [5000, 10000, 15000, 20000, 22500, 25000]:
masks = []
for i in range(class_id, len(probabilities), 4):
probability = probabilities[i]
predict, num_predict = post_process(sigmoid(probability), t, ms)
 masks.append(predict)
 d = []
for i, j in zip(masks, valid_masks[class_id::4]):
if(i.sum() == 0)&(j.sum() == 0):
d.append(1)
 else:
d.append(dice(i, j))
attempts.append(( t, ms, np.mean(d)))
 attempts_df = pd.DataFrame(attempts, columns=['threshold', 'size', 'dice'])
 attempts_df = attempts_df.sort_values('dice', ascending=False)
 print(attempts_df.head())
 best_threshold = attempts_df['threshold'].values[0]
best_size = attempts_df['size'].values[0]
class_params[class_id] =(best_threshold, best_size )
train_df[['Pclass', 'Survived']].groupby(['Pclass'], as_index=False ).mean().sort_values(by='Survived', ascending=False )<sort_values>
del masks
del valid_masks
del probabilities
gc.collect()
train_df[["Sex", "Survived"]].groupby(['Sex'], as_index=False ).mean().sort_values(by='Survived', ascending=False )<sort_values>
print(class_params )
train_df[["SibSp", "Survived"]].groupby(['SibSp'], as_index=False ).mean().sort_values(by='Survived', ascending=False )<sort_values>
attempts_df = pd.DataFrame(attempts, columns=['threshold', 'size', 'dice'] )
train_df[["Parch", "Survived"]].groupby(['Parch'], as_index=False ).mean().sort_values(by='Survived', ascending=False )<feature_engineering>
attempts_df = attempts_df.sort_values('dice', ascending=False)
 attempts_df.head(10 )
for dataset in combine:
dataset['Title'] = dataset.Name.str.extract('([A-Za-z]+)\.', expand=False)
 pd.crosstab(train_df['Title'], train_df['Sex'] )<feature_engineering>
best_threshold = attempts_df['threshold'].values[0]
best_size = attempts_df['size'].values[0]
for dataset in combine:
dataset['Title'] = dataset['Title'].replace(['Lady', 'Countess','Jonkheer', 'Dona'], 'Lady')
 dataset['Title'] = dataset['Title'].replace(['Capt', 'Don', 'Major', 'Sir'], 'Sir')
 dataset['Title'] = dataset['Title'].replace('Mlle', 'Miss')
 dataset['Title'] = dataset['Title'].replace('Ms', 'Miss')
 dataset['Title'] = dataset['Title'].replace('Mme', 'Mrs')
 train_df[['Title', 'Survived']].groupby(['Title'], as_index=False ).mean()<categorify>
for i,(input, output)in enumerate(zip(
 valid_dataset, runner.callbacks[0].predictions["logits"])) :
image, mask = input
image_vis = image.transpose(1, 2, 0)
 mask = mask.astype('uint8' ).transpose(1, 2, 0)
 pr_mask = np.zeros(( 350, 525, 4))
for j in range(4):
probability = cv2.resize(output[:, :, j], dsize=(525, 350), interpolation=cv2.INTER_LINEAR)
 pr_mask[:, :, j], _ = post_process(sigmoid(probability), class_params[j][0], class_params[j][1])
 visualize_with_raw(image=image_vis, mask=pr_mask, original_image=image_vis, original_mask=mask, raw_image=image_vis, raw_mask=output.transpose(1, 2, 0))
if i >= 2:
break
title_mapping = {"Col": 1, "Dr": 2, "Lady": 3, "Master": 4, "Miss": 5, "Mr": 6, "Mrs": 7, "Rev": 8, "Sir": 9}
for dataset in combine:
dataset['Title'] = dataset['Title'].map(title_mapping)
 dataset['Title'] = dataset['Title'].fillna(0)
 train_df.head()<drop_column>
torch.cuda.empty_cache()
gc.collect()
train_df = train_df.drop(['Name', 'PassengerId'], axis=1)
 test_df = test_df.drop(['Name'], axis=1)
 combine = [train_df, test_df]
train_df.shape, test_df.shape<data_type_conversions>
test_dataset = CloudDataset(df=sub, datatype='test', img_ids=test_ids, transforms = get_validation_augmentation() , preprocessing=get_preprocessing(preprocessing_fn))
test_loader = DataLoader(test_dataset, batch_size=8, shuffle=False, num_workers=0)
 loaders = {"test": test_loader}
for dataset in combine:
dataset['Sex'] = dataset['Sex'].map({'female': 1, 'male': 0} ).astype(int)
 train_df.head()<define_variables>
del train_dataset, train_loader
guess_ages = np.zeros(( 2,3))
guess_ages<find_best_params>
del valid_dataset, valid_loader
gc.collect()
for dataset in combine:
for i in range(0, 2):
for j in range(0, 3):
guess_df = dataset[(dataset['Sex'] == i)& \ 
(dataset['Pclass'] == j+1)]['Age'].dropna()
age_guess = guess_df.median()
guess_ages[i,j] = int(age_guess/0.5 + 0.5)* 0.5
for i in range(0, 2):
for j in range(0, 3):
dataset.loc[(dataset.Age.isnull())&(dataset.Sex == i)&(dataset.Pclass == j+1),\
'Age'] = guess_ages[i,j]
dataset['Age'] = dataset['Age'].astype(int)
 train_df.head()<sort_values>
encoded_pixels = []
image_id = 0
for i, test_batch in enumerate(tqdm.tqdm(loaders['test'])) :
runner_out = runner.predict_batch({"features": test_batch[0].cuda() })['logits']
for i, batch in enumerate(runner_out):
for probability in batch:
probability = probability.cpu().detach().numpy()
if probability.shape !=(350, 525):
probability = cv2.resize(probability, dsize=(525, 350), interpolation=cv2.INTER_LINEAR)
 predict, num_predict = post_process(sigmoid(probability), class_params[image_id % 4][0], class_params[image_id % 4][1])
 if num_predict == 0:
encoded_pixels.append('')
 else:
r = mask2rle(predict)
 encoded_pixels.append(r)
 image_id += 1
<feature_engineering><EOS> 
sub['EncodedPixels'] = encoded_pixels
sub.to_csv('submission.csv', columns=['Image_Label', 'EncodedPixels'], index=False )