Kaggle competition: TGS Salt Identification Challenge
<SOS> metric: IntersectionOverUnionObjectSegmentation Kaggle data source: tgs-salt-identification-challenge<load_from_csv>
plt.style.use('seaborn-white')
 sns.set_style("white")
train_data = pd.read_csv('.. /input/train.csv')
 test_data = pd.read_csv('.. /input/test.csv')
 test_data['Survived'] = np.nan
df = pd.concat([train_data, test_data], ignore_index=True, sort=False)
 df.info()
sns.barplot(x='Sex', y='Survived', data=df, palette='Set3')
 plt.show()
age_df = df[['Age', 'Pclass','Sex','Parch','SibSp']]
age_df=pd.get_dummies(age_df)
 known_age = age_df[age_df.Age.notnull() ].values
unknown_age = age_df[age_df.Age.isnull() ].values
X = known_age[:, 1:]
y = known_age[:, 0]
rfr = RandomForestRegressor(random_state=0, n_estimators=100, n_jobs=-1)
 rfr.fit(X, y)
 predictedAges = rfr.predict(unknown_age[:, 1::])
 df.loc[(df.Age.isnull()), 'Age'] = predictedAges
facet = sns.FacetGrid(df[0:890], hue="Survived",aspect=2)
 facet.map(sns.kdeplot,'Age',shade= True)
 facet.set(xlim=(0, df.loc[0:890,'Age'].max()))
facet.add_legend()
plt.show()
df['Title'] = df['Name'].map(lambda x: x.split(', ')[1].split('.')[0])
 df['Title'].replace(['Capt', 'Col', 'Major', 'Dr', 'Rev'], 'Officer', inplace=True)
 df['Title'].replace(['Don', 'Sir', 'the Countess', 'Lady', 'Dona'], 'Royalty', inplace=True)
 df['Title'].replace(['Mme', 'Ms'], 'Mrs', inplace=True)
 df['Title'].replace(['Mlle'], 'Miss', inplace=True)
 df['Title'].replace(['Jonkheer'], 'Master', inplace=True)
 sns.barplot(x='Title', y='Survived', data=df, palette='Set3')
 df['Surname'] = df['Name'].map(lambda name:name.split(',')[0].strip())
 df['FamilyGroup'] = df['Surname'].map(df['Surname'].value_counts())
 Female_Child_Group=df.loc[(df['FamilyGroup']>=2)&(( df['Age']<=16)|(df['Sex']=='female')) ]
Female_Child_Group=Female_Child_Group.groupby('Surname')['Survived'].mean()
print(Female_Child_Group.value_counts())
 Male_Adult_Group=df.loc[(df['FamilyGroup']>=2)&(df['Age']>16)&(df['Sex']=='male')]
Male_Adult_List=Male_Adult_Group.groupby('Surname')['Survived'].mean()
print(Male_Adult_List.value_counts())
 Dead_list=set(Female_Child_Group[Female_Child_Group.apply(lambda x:x==0)].index)
 Survived_list=set(Male_Adult_List[Male_Adult_List.apply(lambda x:x==1)].index)
 print('Dead_list = ', Dead_list)
 print('Survived_list = ', Survived_list)
 df.loc[(df['Survived'].isnull())&(df['Surname'].apply(lambda x:x in Dead_list)) ,\
['Sex','Age','Title']] = ['male',28.0,'Mr']
df.loc[(df['Survived'].isnull())&(df['Surname'].apply(lambda x:x in Survived_list)) ,\
['Sex','Age','Title']] = ['female',5.0,'Mrs']
fare=df.loc[(df['Embarked'] == 'S')&(df['Pclass'] == 3), 'Fare'].median()
df['Fare']=df['Fare'].fillna(fare)
 df['Family']=df['SibSp']+df['Parch']+1
df.loc[(df['Family']>=2)&(df['Family']<=4), 'Family_label'] = 2
df.loc[(df['Family']>=5)&(df['Family']<=7)|(df['Family']==1), 'Family_label'] = 1
df.loc[(df['Family']>=8), 'Family_label'] = 0
Ticket_Count = dict(df['Ticket'].value_counts())
 df['TicketGroup'] = df['Ticket'].map(Ticket_Count)
 sns.barplot(x='TicketGroup', y='Survived', data=df, palette='Set3')
 plt.show()
df.loc[(df['TicketGroup']>=2)&(df['TicketGroup']<=4), 'Ticket_label'] = 2
df.loc[(df['TicketGroup']>=5)&(df['TicketGroup']<=8)|(df['TicketGroup']==1), 'Ticket_label'] = 1
df.loc[(df['TicketGroup']>=11), 'Ticket_label'] = 0
sns.barplot(x='Ticket_label', y='Survived', data=df, palette='Set3')
 plt.show()
df['Cabin'] = df['Cabin'].fillna('Unknown')
 df['Cabin_label']=df['Cabin'].str.get(0)
 sns.barplot(x='Cabin_label', y='Survived', data=df, palette='Set3')
 plt.show()
df['Embarked'] = df['Embarked'].fillna('S')
 df = df[['Survived','Pclass','Sex','Age','Fare','Embarked','Title','Family_label','Cabin_label','Ticket_label']]
df = pd.get_dummies(df)
 train = df[df['Survived'].notnull() ]
test = df[df['Survived'].isnull() ].drop('Survived',axis=1)
 X = train.values[:,1:]
y = train.values[:,0]
test_x = test.values
select = SelectKBest(k = 20)
 clf = RandomForestClassifier(random_state = 10,
warm_start = True,
n_estimators = 26,
max_depth = 6,
max_features = 'sqrt')
 pipeline = make_pipeline(select, clf)
 pipeline.fit(X, y)
 cv_result = cross_validate(pipeline, X, y, cv= 10)
 print('mean_score = ', np.mean(cv_result['test_score']))
print('mean_std = ', np.std(cv_result['test_score']))
mask= select.get_support()
list_col = list(df.columns[1:])
 for i, j in enumerate(list_col):
print('No'+str(i+1), j,'=', mask[i])
 X_selected = select.transform(X)
 print('X.shape={}, X_selected.shape={}'.format(X.shape, X_selected.shape))
PassengerId=test_data['PassengerId']
predictions = pipeline.predict(test_x)
 submission = pd.DataFrame({"PassengerId": PassengerId, "Survived": predictions.astype(np.int32)})
 submission.to_csv("my_submission.csv", index=False )<import_modules>
img_size_ori = 101
img_size_target = 128
def upsample(img):
if img_size_ori == img_size_target:
return img
return resize(img,(img_size_target, img_size_target), mode='constant', preserve_range=True)
 def downsample(img):
if img_size_ori == img_size_target:
return img
return resize(img,(img_size_ori, img_size_ori), mode='constant', preserve_range=True)
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename)) 
<load_from_csv>
class CyclicLR(Callback):

 def __init__(self, base_lr=0.001, max_lr=0.006, step_size=2000., mode='triangular',
gamma=1., scale_fn=None, scale_mode='cycle'):
super(CyclicLR, self ).__init__()
self.base_lr = base_lr
self.max_lr = max_lr
self.step_size = step_size
self.mode = mode
self.gamma = gamma
if scale_fn == None:
if self.mode == 'triangular':
self.scale_fn = lambda x: 1.
 self.scale_mode = 'cycle'
elif self.mode == 'triangular2':
self.scale_fn = lambda x: 1/(2.**(x-1))
self.scale_mode = 'cycle'
elif self.mode == 'exp_range':
self.scale_fn = lambda x: gamma**(x)
 self.scale_mode = 'iterations'
else:
self.scale_fn = scale_fn
self.scale_mode = scale_mode
self.clr_iterations = 0.
 self.trn_iterations = 0.
 self.history = {}
self._reset()
def _reset(self, new_base_lr=None, new_max_lr=None,
new_step_size=None):

 if new_base_lr != None:
self.base_lr = new_base_lr
if new_max_lr != None:
self.max_lr = new_max_lr
if new_step_size != None:
self.step_size = new_step_size
self.clr_iterations = 0.
 def clr(self):
cycle = np.floor(1+self.clr_iterations/(2*self.step_size))
x = np.abs(self.clr_iterations/self.step_size - 2*cycle + 1)
 if self.scale_mode == 'cycle':
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(cycle)
 else:
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(self.clr_iterations)
 def on_train_begin(self, logs={}):
logs = logs or {}
if self.clr_iterations == 0:
K.set_value(self.model.optimizer.lr, self.base_lr)
 else:
K.set_value(self.model.optimizer.lr, self.clr())
 def on_batch_end(self, epoch, logs=None):
logs = logs or {}
self.trn_iterations += 1
self.clr_iterations += 1
self.history.setdefault('lr', [] ).append(K.get_value(self.model.optimizer.lr))
self.history.setdefault('iterations', [] ).append(self.trn_iterations)
 for k, v in logs.items() :
self.history.setdefault(k, [] ).append(v)
 K.set_value(self.model.optimizer.lr, self.clr())
train_data = pd.read_csv('/kaggle/input/titanic/train.csv')
 test_data = pd.read_csv('/kaggle/input/titanic/test.csv' )<feature_engineering>
train_df = pd.read_csv(".. /input/train.csv", index_col="id", usecols=[0])
 depths_df = pd.read_csv(".. /input/depths.csv", index_col="id")
 train_df = train_df.join(depths_df)
 test_df = depths_df[~depths_df.index.isin(train_df.index)]
train_data['FamilySize'] = 1+train_data['Parch']+train_data['SibSp']
test_data['FamilySize'] = 1+test_data['Parch']+test_data['SibSp']<count_values>
train_df["images"] = [np.array(load_img(".. /input/train/images/{}.png".format(idx), grayscale=True)) / 255 for idx in tqdm_notebook(train_df.index)]
train_data['Age'].fillna(train_data['Age'].mean() , inplace=True)
 train_data['Embarked'].value_counts()
train_data['Embarked'].fillna('S',inplace=True)
 train_data.apply(lambda x: sum(x.isnull()),axis=0 )<count_values>
train_df["masks"] = [np.array(load_img(".. /input/train/masks/{}.png".format(idx), grayscale=True)) / 255 for idx in tqdm_notebook(train_df.index)]
test_data['Age'].fillna(test_data['Age'].mean() , inplace=True)
 test_data['Fare'].fillna(test_data['Fare'].mean() , inplace=True)
 test_data['Embarked'].value_counts()
test_data['Embarked'].fillna('S',inplace=True)
 test_data.apply(lambda x: sum(x.isnull()),axis=0 )<categorify>
train_df["coverage"] = train_df.masks.map(np.sum)/ pow(img_size_ori, 2 )
features = ["Pclass", "Sex", "SibSp", "Parch", "Embarked"]
train_data = pd.get_dummies(train_data, columns=features, prefix = features)
 test_data = pd.get_dummies(test_data, columns=features, prefix = features )<normalization>
def cov_to_class(val):
for i in range(0, 11):
if val * 10 <= i :
return i
train_df["coverage_class"] = train_df.coverage.map(cov_to_class )
train_data[['normalized_Age', 'normalized_Fare']] = preprocessing.StandardScaler().fit_transform(train_data[['Age', 'Fare']])
 test_data[['normalized_Age', 'normalized_Fare']] = preprocessing.StandardScaler().fit_transform(test_data[['Age', 'Fare']] )<prepare_x_and_y>
ids_train, ids_valid, x_train, x_valid, y_train, y_valid, cov_train, cov_test, depth_train, depth_test = train_test_split(
 train_df.index.values,
np.array(train_df.images.map(upsample ).tolist() ).reshape(-1, img_size_target, img_size_target, 1),
np.array(train_df.masks.map(upsample ).tolist() ).reshape(-1, img_size_target, img_size_target, 1),
train_df.coverage.values,
train_df.z.values,
test_size=0.2, stratify=train_df.coverage_class, random_state=1337 )
bestfeatures = SelectKBest(score_func=chi2, k=10)
 X = train_data[['Age', 'Fare', 'FamilySize','Pclass_1', 'Pclass_2',
'Pclass_3', 'Sex_female', 'Sex_male', 'SibSp_0', 'SibSp_1', 'SibSp_2',
'SibSp_3', 'SibSp_4', 'SibSp_5', 'SibSp_8', 'Parch_0', 'Parch_1',
'Parch_2', 'Parch_3', 'Parch_4', 'Parch_5', 'Parch_6', 'Embarked_C',
'Embarked_Q', 'Embarked_S']]
y = train_data['Survived']
fit = bestfeatures.fit(X,y)
 dfscores = pd.DataFrame(fit.scores_)
 dfcolumns = pd.DataFrame(X.columns)
 featureScores = pd.concat([dfcolumns,dfscores],axis=1)
 featureScores.columns = ['Specs','Score']
print(featureScores.nlargest(10,'Score'))<categorify>
features = ["AgeBin", "FareBin", "FamilyBin"]
train_data = pd.get_dummies(train_data, columns=features, prefix = features)
 test_data = pd.get_dummies(test_data, columns=features, prefix = features )<prepare_x_and_y>
def conv_block(m, dim, acti, bn, res, do=0.5):
n = Conv2D(dim, 3, activation=acti, padding='same' )(m)
 n = BatchNormalization()(n)if bn else n
n = SpatialDropout2D(do )(n)if do else n
n = Conv2D(dim, 3, activation=acti, padding='same' )(n)
 n = BatchNormalization()(n)if bn else n
return Concatenate()([m, n])if res else n
def level_block(m, dim, depth, inc, acti, do, bn, mp, up, res):
if depth > 0:
n = conv_block(m, dim, acti, bn, res)
 m = MaxPooling2D()(n)if mp else Conv2D(dim, 3, strides=2, padding='same' )(n)
 m = level_block(m, int(inc*dim), depth-1, inc, acti, do, bn, mp, up, res)
 if up:
m = UpSampling2D()(m)
 m = Conv2D(dim, 2, activation=acti, padding='same' )(m)
 else:
m = Conv2DTranspose(dim, 3, strides=2, activation=acti, padding='same' )(m)
 n = Concatenate()([n, m])
 m = conv_block(n, dim, acti, bn, res)
 else:
m = conv_block(m, dim, acti, bn, res, do=do)
 return m
def UNet(img_shape, out_ch=1, start_ch=64, depth=5, inc_rate=2., activation='relu',
dropout=0.5, batchnorm=False, maxpool=True, upconv=True, residual=True):
i = Input(shape=img_shape)
 o = level_block(i, start_ch, depth, inc_rate, activation, dropout, batchnorm, maxpool, upconv, residual)
 o = Conv2D(out_ch, 1, activation='sigmoid' )(o)
 return Model(inputs=i, outputs=o )
X = train_data[['Pclass_1', 'Pclass_2', 'Pclass_3', 'Sex_female', 'Sex_male', 'Embarked_C', 'Embarked_Q', 'Embarked_S', 'AgeBin_1',
'AgeBin_2', 'AgeBin_3', 'AgeBin_4', 'AgeBin_5', 'FareBin_1',
'FareBin_2', 'FareBin_3', 'FareBin_4', 'FareBin_5', 'FamilyBin_1',
'FamilyBin_2', 'FamilyBin_3', 'FamilyBin_4']]
y = train_data['Survived']
k_fold = KFold(n_splits=10)
 classifier = RandomForestClassifier(n_estimators=100, max_depth=5, random_state=0)
 results_kfold_CV = cross_val_score(classifier, X, y, cv=k_fold, n_jobs=-1)
 results_kfold_CV.mean() *100<drop_column>
model = UNet(( img_size_target,img_size_target,1),start_ch=16,depth=5,batchnorm=True )
X = train_data[['Pclass_1', 'Pclass_2', 'Pclass_3', 'Sex_female', 'Sex_male', 'SibSp_0',
'SibSp_1', 'SibSp_2', 'SibSp_3', 'SibSp_4', 'SibSp_5', 'SibSp_8',
'Parch_0', 'Parch_1', 'Parch_2', 'Parch_3', 'Parch_4', 'Parch_5',
'Parch_6', 'Embarked_C', 'Embarked_Q', 'Embarked_S', 'AgeBin_1',
'AgeBin_2', 'AgeBin_3', 'AgeBin_4', 'AgeBin_5', 'FareBin_1',
'FareBin_2', 'FareBin_3', 'FareBin_4', 'FareBin_5']]
y = train_data['Survived']
k_fold = KFold(n_splits=10)
 classifier = RandomForestClassifier(n_estimators=100, max_depth=5, random_state=0)
 results_kfold_CV = cross_val_score(classifier, X, y, cv=k_fold, n_jobs=-1)
 results_kfold_CV.mean() *100 
<compute_train_metric>
def iou_loss(y_true, y_pred):
y_true = tf.reshape(y_true, [-1])
 y_pred = tf.reshape(y_pred, [-1])
 intersection = tf.reduce_sum(y_true * y_pred)
 score =(intersection + 1.) /(tf.reduce_sum(y_true)+ tf.reduce_sum(y_pred)- intersection + 1.)
return 1 - score
def iou_bce_loss(y_true, y_pred):
return 0.5 * keras.losses.binary_crossentropy(y_true, y_pred)+ 0.5 * iou_loss(y_true, y_pred)
 def mean_iou(y_true, y_pred):
y_pred = tf.round(y_pred)
 intersect = tf.reduce_sum(y_true * y_pred, axis=[1, 2, 3])
 union = tf.reduce_sum(y_true, axis=[1, 2, 3])+ tf.reduce_sum(y_pred, axis=[1, 2, 3])
 smooth = tf.ones(tf.shape(intersect))
return tf.reduce_mean(( intersect + smooth)/(union - intersect + smooth))
model.compile(loss=iou_bce_loss, optimizer="adam", metrics=['accuracy', mean_iou] )
classifier = XGBClassifier()
results_kfold_CV = cross_val_score(classifier, X, y, cv=k_fold, n_jobs=-1)
 results_kfold_CV.mean() *100<compute_train_metric>
Vx = [np.flip(x, axis=0)for x in x_train]
Vy = [np.flip(x, axis=0)for x in y_train]
Hx = [np.flip(x, axis=1)for x in x_train]
Hy = [np.flip(x, axis=1)for x in y_train]
HVx = [np.flip(x, axis=1)for x in Vx]
HVy = [np.flip(x, axis=1)for x in Vy]
x_train = np.append(x_train, Vx, axis=0)
 y_train = np.append(y_train, Vy, axis=0)
 x_train = np.append(x_train, Hx, axis=0)
 y_train = np.append(y_train, Hy, axis=0)
 x_train = np.append(x_train, HVx, axis=0)
 y_train = np.append(y_train, HVy, axis=0 )
classifier = DecisionTreeClassifier(max_depth=5)
 results_kfold_CV = cross_val_score(classifier, X, y, cv=k_fold, n_jobs=-1)
 results_kfold_CV.mean() *100<train_model>
early_stopping = EarlyStopping(patience=10, verbose=1)
 model_checkpoint = ModelCheckpoint("./keras.model", save_best_only=True, verbose=1)
 reduce_lr = ReduceLROnPlateau(factor=0.1, patience=5, min_lr=0.0000005, verbose=1)
 clr = CyclicLR(base_lr=0.001, max_lr=0.0001, step_size=200, mode='triangular')
 def cosine_annealing(x):
lr = 0.001
epochs = 200
return lr*(np.cos(np.pi*x/epochs)+1.) /2
learning_rate = tf.keras.callbacks.LearningRateScheduler(cosine_annealing)
 epochs = 200
batch_size = 32
history = model.fit(x_train, y_train,
validation_data=[x_valid, y_valid],
epochs=epochs,
batch_size=batch_size,
callbacks=[early_stopping, model_checkpoint, learning_rate] )
clf = RandomForestClassifier(n_estimators=200, bootstrap=False, criterion='gini',min_samples_leaf = 1,
min_samples_split=2, max_depth=5, random_state=42 ).fit(X,y)
 clf.score(X,y )<compute_train_metric>
preds_valid = model.predict(x_valid ).reshape(-1, img_size_target, img_size_target)
 preds_valid = np.array([downsample(x)for x in preds_valid])
 mask_valid = np.array([downsample(x)for x in y_valid] )
k_fold = KFold(n_splits=10)
 classifier = RandomForestClassifier(n_estimators=200, bootstrap=False, criterion='gini',min_samples_leaf = 1,
min_samples_split=2, max_depth=5, random_state=42)
 results_kfold_CV = cross_val_score(classifier, X, y, cv=k_fold, n_jobs=-1)
 results_kfold_CV.mean() *100<train_model>
thresholds = np.linspace(0, 1, 50)
 ious = np.array([iou_metric_batch(mask_valid, np.int32(preds_valid > threshold)) for threshold in tqdm_notebook(thresholds)] )
clf = XGBClassifier(n_estimators=70, eta=0.06, gamma=0.1, max_depth = 5, objective = 'binary:logistic', reg_lambda = 2 ).fit(X,y)
 clf.score(X,y )<compute_train_metric>
threshold_best_index = np.argmax(ious[9:-10])+ 9
iou_best = ious[threshold_best_index]
threshold_best = thresholds[threshold_best_index]
k_fold = KFold(n_splits=10)
 classifier = XGBClassifier(n_estimators=70, eta=0.06, gamma=0.1, max_depth = 5, objective = 'binary:logistic', reg_lambda = 2)
 results_kfold_CV = cross_val_score(classifier, X, y, cv=k_fold, n_jobs=-1)
 results_kfold_CV.mean() *100<drop_column>
def RLenc(img, order='F', format=True):

 bytes = img.reshape(img.shape[0] * img.shape[1], order=order)
 runs = []
r = 0
pos = 1
for c in bytes:
if(c == 0):
if r != 0:
runs.append(( pos, r))
pos += r
r = 0
pos += 1
else:
r += 1
if r != 0:
runs.append(( pos, r))
pos += r
r = 0
if format:
z = ''
for rr in runs:
z += '{} {} '.format(rr[0], rr[1])
 return z[:-1]
else:
return runs
X_test = test_data[['Pclass_1', 'Pclass_2', 'Pclass_3', 'Sex_female', 'Sex_male', 'SibSp_0',
'SibSp_1', 'SibSp_2', 'SibSp_3', 'SibSp_4', 'SibSp_5', 'SibSp_8',
'Parch_0', 'Parch_1', 'Parch_2', 'Parch_3', 'Parch_4', 'Parch_5',
'Parch_6', 'Embarked_C', 'Embarked_Q', 'Embarked_S', 'AgeBin_1',
'AgeBin_2', 'AgeBin_3', 'AgeBin_4', 'AgeBin_5', 'FareBin_1',
'FareBin_2', 'FareBin_3', 'FareBin_4', 'FareBin_5']]
pred = clf.predict(X_test )<save_to_csv>
preds_test = model.predict(x_test, verbose=True )
testDF = pd.DataFrame()
testDF['PassengerId'] = test_data['PassengerId']
testDF['Survived'] = pred
testDF.to_csv('submission.csv', index=False )<load_from_csv>
pred_dict = {idx: RLenc(np.round(downsample(preds_test[i])> threshold_best)) for i, idx in enumerate(tqdm_notebook(test_df.index.values)) }
<count_missing_values><EOS> 
def create_download_link(df, title = "Download CSV file", filename = "sub.csv"):
csv = df.to_csv()
b64 = base64.b64encode(csv.encode())
 payload = b64.decode()
html = '<a download="{filename}" href="data:text/csv;base64,{payload}" target="_blank">{title}</a>'
html = html.format(payload=payload,title=title,filename=filename)
 return HTML(html)
 sub = pd.DataFrame.from_dict(pred_dict,orient='index')
 sub.index.names = ['id']
sub.columns = ['rle_mask']
sub.to_csv('submission.csv')
 timestamp = datetime.datetime.now().strftime("%Y%m%d-%H%M")
 print('Submission output to: sub-{}.csv'.format(timestamp))
sub.to_csv("sub-{}.csv".format(timestamp))
create_download_link(sub)