Kaggle competition: TGS Salt Identification Challenge
<SOS> metric: IntersectionOverUnionObjectSegmentation Kaggle data source: tgs-salt-identification-challenge<save_to_csv>
plt.style.use('seaborn-white')
 sns.set_style("white")
 %matplotlib inline
t_start = time.time()
predictions = pd.DataFrame({"PassengerId" : list(range(892, 892 + len(y_pred))),"Survived" : y_pred})
 predictions.to_csv('predictions.csv', index = False )<import_modules>
version = 5
basic_name = f'Unet_resnet_v{version}'
save_model_name = basic_name + '.model'
previous_model_name = ".. /input/u-net-with-simple-resnet-blocks-v2-new-loss/" + save_model_name
submission_file = basic_name + '.csv'
data_source = ".. /input/tgs-salt-identification-challenge/"
stored_trained_model = ".. /input/trained-unet-model/Unet_resnet_v5.model"
fMosaic = ".. /input/trained-unet-model/mosaic.csv"
fTrain = ".. /input/trained-unet-model/train-files.csv"
fTest = ".. /input/trained-unet-model/test-files.csv"
import numpy as np
import pandas as pd
import os
from sklearn.model_selection import train_test_split<load_from_csv>
img_size_ori = 101
img_size_target = 101
def upsample(img):
if img_size_ori == img_size_target:
return img
return resize(img,(img_size_target, img_size_target), mode='constant', preserve_range=True)
 def downsample(img):
if img_size_ori == img_size_target:
return img
return resize(img,(img_size_ori, img_size_ori), mode='constant', preserve_range=True )
data_root = '/kaggle/input/titanic'
train_data_path = os.path.join(data_root, 'train.csv')
 test_data_path = os.path.join(data_root, 'test.csv')
 train_data = pd.read_csv(train_data_path)
 test_data = pd.read_csv(test_data_path )<define_variables>
mosaic_df=pd.read_csv(fMosaic)
 train_files_df = pd.read_csv(fTrain)
 test_files_df = pd.read_csv(fTest)
 train_files_df.insert(0, 'img_id', range(0,len(train_files_df)))
 test_files_df.insert(0, 'img_id', range(len(train_files_df),len(train_files_df)+len(test_files_df)))
 all_files_df = pd.concat([train_files_df,test_files_df],sort=False)
 all_files_df[['img_name','a','b']]=all_files_df['x'].str.partition(".")
use_cols = ['Pclass', 'Sex', 'SibSp', 'Parch']<prepare_x_and_y>
train_df = pd.read_csv(data_source + "train.csv", index_col="id", usecols=[0])
 depths_df = pd.read_csv(data_source + "depths.csv", index_col="id")
 train_df = train_df.join(depths_df)
 test_df = depths_df[~depths_df.index.isin(train_df.index)]
learn_data, valid_data = train_test_split(train_data, test_size=0.3, random_state=0)
 X = pd.get_dummies(learn_data.loc[:, use_cols] ).values
y = learn_data.loc[:, 'Survived'].values
valid_X = pd.get_dummies(valid_data.loc[:, use_cols] ).values
valid_y = valid_data.loc[:, 'Survived'].values<import_modules>
test_df["images"] = [np.array(load_img(data_source + "test/images/{}.png".format(idx), grayscale=True)) / 255 for idx in tqdm_notebook(test_df.index)]
from xgboost import XGBClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.tree import DecisionTreeClassifier<choose_model_class>
train_df["images"] = [np.array(load_img(data_source + "train/images/{}.png".format(idx), grayscale=True)) / 255 for idx in tqdm_notebook(train_df.index)]
cls = DecisionTreeClassifier()
cls.fit(X, y)
 y_pred = cls.predict(valid_X)
 y_true = valid_y
print(f1_score(y_true=y_true, y_pred=y_pred))
print(accuracy_score(y_true=y_true, y_pred=y_pred))<save_to_csv>
train_df["masks"] = [np.array(load_img(data_source + "train/masks/{}.png".format(idx), grayscale=True)) / 255 for idx in tqdm_notebook(train_df.index)]
test_X = pd.get_dummies(test_data.loc[:, use_cols] ).values
test_y_pred = cls.predict(test_X)
 predict_df = pd.DataFrame()
predict_df['PassengerId'] = test_data['PassengerId']
predict_df['Survived'] = test_y_pred
predict_df.to_csv('submission.csv', index=False )<load_from_csv>
train=pd.read_csv('/kaggle/input/titanic/train.csv')
 test=pd.read_csv('/kaggle/input/titanic/test.csv')
 target=train['Survived']
submission=pd.DataFrame(test['PassengerId'] )<categorify>
all_arr = pd.concat([train_df,test_df],sort=False)
 all_arr.insert(0, 'orig_id', range(1,len(all_arr)+1))
a=dataset.groupby('Pclass')['Age'].median()
dataset['Age']=dataset['Age'].fillna(dataset['Pclass'].map(a))
a=dataset.groupby('Pclass')['Fare'].median()
dataset['Fare']=dataset['Fare'].fillna(dataset['Pclass'].map(a))
dataset['Embarked'].fillna('S',inplace=True)
<feature_engineering>
all_arr = pd.merge(all_arr,all_files_df,how = 'left',left_index=True, right_on = 'img_name')
dataset['Name'].iloc[3].split() [1]
a=[]
for i in range(len(dataset)) :
a.append(dataset['Name'].iloc[i].split() [1])
 a=pd.Series(a)
 dataset['Title']=a 
<feature_engineering>
train_df["coverage"] = train_df.masks.map(np.sum)/ pow(img_size_ori, 2)
 def cov_to_class(val):
for i in range(0, 11):
if val * 10 <= i :
return i
train_df["coverage_class"] = train_df.coverage.map(cov_to_class)
dataset['Passenger']=dataset['SibSp']+dataset['Parch']+1
dataset 
<choose_model_class>
ids_train, ids_valid, x_train, x_valid, y_train, y_valid, cov_train, cov_test, depth_train, depth_test = train_test_split(
 train_df.index.values,
np.array(train_df.images.map(upsample ).tolist() ).reshape(-1, img_size_target, img_size_target, 1),
np.array(train_df.masks.map(upsample ).tolist() ).reshape(-1, img_size_target, img_size_target, 1),
train_df.coverage.values,
train_df.z.values,
test_size=0.2, stratify=train_df.coverage_class, random_state= None )
sky=LogisticRegression()
leaks = {
897:1,
899:1,
930:1,
932:1,
949:1,
987:1,
995:1,
998:1,
999:1,
1016:1,
1047:1,
1083:1,
1097:1,
1099:1,
1103:1,
1115:1,
} 
<save_to_csv>
def BatchActivate(x):
x = BatchNormalization()(x)
 x = Activation('relu' )(x)
 return x
def convolution_block(x, filters, size, strides=(1,1), padding='same', activation=True):
x = Conv2D(filters, size, strides=strides, padding=padding )(x)
 if activation == True:
x = BatchActivate(x)
 return x
def residual_block(blockInput, num_filters=16, batch_activate = False):
x = BatchActivate(blockInput)
 x = convolution_block(x, num_filters,(3,3))
x = convolution_block(x, num_filters,(3,3), activation=False)
 x = Add()([x, blockInput])
 if batch_activate:
x = BatchActivate(x)
 return x
sky.fit(train,target)
 a=sky.predict(test)
 submission['Survived']=a
submission['Survived'] = submission['Survived'].apply(lambda x: 1 if x>0.8 else 0)
 submission['Survived'] = submission.apply(lambda r: leaks[int(r['PassengerId'])] if int(r['PassengerId'])in leaks else r['Survived'], axis=1)
 submission.to_csv('sub_tit.csv', index=False )<set_options>
def build_model(input_layer, start_neurons, DropoutRatio = 0.5):
conv1 = Conv2D(start_neurons * 1,(3, 3), activation=None, padding="same" )(input_layer)
 conv1 = residual_block(conv1,start_neurons * 1)
 conv1 = residual_block(conv1,start_neurons * 1, True)
 pool1 = MaxPooling2D(( 2, 2))(conv1)
 pool1 = Dropout(DropoutRatio/2 )(pool1)
 conv2 = Conv2D(start_neurons * 2,(3, 3), activation=None, padding="same" )(pool1)
 conv2 = residual_block(conv2,start_neurons * 2)
 conv2 = residual_block(conv2,start_neurons * 2, True)
 pool2 = MaxPooling2D(( 2, 2))(conv2)
 pool2 = Dropout(DropoutRatio )(pool2)
 conv3 = Conv2D(start_neurons * 4,(3, 3), activation=None, padding="same" )(pool2)
 conv3 = residual_block(conv3,start_neurons * 4)
 conv3 = residual_block(conv3,start_neurons * 4, True)
 pool3 = MaxPooling2D(( 2, 2))(conv3)
 pool3 = Dropout(DropoutRatio )(pool3)
 conv4 = Conv2D(start_neurons * 8,(3, 3), activation=None, padding="same" )(pool3)
 conv4 = residual_block(conv4,start_neurons * 8)
 conv4 = residual_block(conv4,start_neurons * 8, True)
 pool4 = MaxPooling2D(( 2, 2))(conv4)
 pool4 = Dropout(DropoutRatio )(pool4)
 convm = Conv2D(start_neurons * 16,(3, 3), activation=None, padding="same" )(pool4)
 convm = residual_block(convm,start_neurons * 16)
 convm = residual_block(convm,start_neurons * 16, True)
 deconv4 = Conv2DTranspose(start_neurons * 8,(3, 3), strides=(2, 2), padding="same" )(convm)
 uconv4 = concatenate([deconv4, conv4])
 uconv4 = Dropout(DropoutRatio )(uconv4)
 uconv4 = Conv2D(start_neurons * 8,(3, 3), activation=None, padding="same" )(uconv4)
 uconv4 = residual_block(uconv4,start_neurons * 8)
 uconv4 = residual_block(uconv4,start_neurons * 8, True)
 deconv3 = Conv2DTranspose(start_neurons * 4,(3, 3), strides=(2, 2), padding="valid" )(uconv4)
 uconv3 = concatenate([deconv3, conv3])
 uconv3 = Dropout(DropoutRatio )(uconv3)
 uconv3 = Conv2D(start_neurons * 4,(3, 3), activation=None, padding="same" )(uconv3)
 uconv3 = residual_block(uconv3,start_neurons * 4)
 uconv3 = residual_block(uconv3,start_neurons * 4, True)
 deconv2 = Conv2DTranspose(start_neurons * 2,(3, 3), strides=(2, 2), padding="same" )(uconv3)
 uconv2 = concatenate([deconv2, conv2])
 uconv2 = Dropout(DropoutRatio )(uconv2)
 uconv2 = Conv2D(start_neurons * 2,(3, 3), activation=None, padding="same" )(uconv2)
 uconv2 = residual_block(uconv2,start_neurons * 2)
 uconv2 = residual_block(uconv2,start_neurons * 2, True)
 deconv1 = Conv2DTranspose(start_neurons * 1,(3, 3), strides=(2, 2), padding="valid" )(uconv2)
 uconv1 = concatenate([deconv1, conv1])
 uconv1 = Dropout(DropoutRatio )(uconv1)
 uconv1 = Conv2D(start_neurons * 1,(3, 3), activation=None, padding="same" )(uconv1)
 uconv1 = residual_block(uconv1,start_neurons * 1)
 uconv1 = residual_block(uconv1,start_neurons * 1, True)
 output_layer_noActi = Conv2D(1,(1,1), padding="same", activation=None )(uconv1)
 output_layer = Activation('sigmoid' )(output_layer_noActi)
 return output_layer
mpl.rcParams.update({'font.size': 13} )<load_from_csv>
def get_iou_vector(A, B):
batch_size = A.shape[0]
metric = []
for batch in range(batch_size):
t, p = A[batch]>0, B[batch]>0
intersection = np.logical_and(t, p)
 union = np.logical_or(t, p)
 iou =(np.sum(intersection > 0)+ 1e-10)/(np.sum(union > 0)+ 1e-10)
 thresholds = np.arange(0.5, 1, 0.05)
 s = []
for thresh in thresholds:
s.append(iou > thresh)
 metric.append(np.mean(s))
return np.mean(metric)
 def my_iou_metric(label, pred):
return tf.py_func(get_iou_vector, [label, pred>0.5], tf.float64)
 def my_iou_metric_2(label, pred):
return tf.py_func(get_iou_vector, [label, pred >0], tf.float64 )
folder = '/kaggle/input/titanic/'
train = pd.read_csv(folder + 'train.csv')
 train_shape = train.shape
test = pd.read_csv(folder + 'test.csv')
 test_shape = test.shape
target = 'survived'
train.columns = train.columns.str.lower()
test.columns = test.columns.str.lower()<count_unique_values>
def lovasz_grad(gt_sorted):

 gts = tf.reduce_sum(gt_sorted)
 intersection = gts - tf.cumsum(gt_sorted)
 union = gts + tf.cumsum(1.- gt_sorted)
 jaccard = 1.- intersection / union
jaccard = tf.concat(( jaccard[0:1], jaccard[1:] - jaccard[:-1]), 0)
 return jaccard
def lovasz_hinge(logits, labels, per_image=True, ignore=None):

 if per_image:
def treat_image(log_lab):
log, lab = log_lab
log, lab = tf.expand_dims(log, 0), tf.expand_dims(lab, 0)
 log, lab = flatten_binary_scores(log, lab, ignore)
 return lovasz_hinge_flat(log, lab)
 losses = tf.map_fn(treat_image,(logits, labels), dtype=tf.float32)
 loss = tf.reduce_mean(losses)
 else:
loss = lovasz_hinge_flat(*flatten_binary_scores(logits, labels, ignore))
return loss
def lovasz_hinge_flat(logits, labels):

 def compute_loss() :
labelsf = tf.cast(labels, logits.dtype)
 signs = 2.* labelsf - 1.
 errors = 1.- logits * tf.stop_gradient(signs)
 errors_sorted, perm = tf.nn.top_k(errors, k=tf.shape(errors)[0], name="descending_sort")
 gt_sorted = tf.gather(labelsf, perm)
 grad = lovasz_grad(gt_sorted)
 loss = tf.tensordot(tf.nn.relu(errors_sorted), tf.stop_gradient(grad), 1, name="loss_non_void")
 return loss
loss = tf.cond(tf.equal(tf.shape(logits)[0], 0),
lambda: tf.reduce_sum(logits)* 0.,
compute_loss,
strict=True,
name="loss" 
)
 return loss
def flatten_binary_scores(scores, labels, ignore=None):

 scores = tf.reshape(scores,(-1,))
labels = tf.reshape(labels,(-1,))
if ignore is None:
return scores, labels
valid = tf.not_equal(labels, ignore)
 vscores = tf.boolean_mask(scores, valid, name='valid_scores')
 vlabels = tf.boolean_mask(labels, valid, name='valid_labels')
 return vscores, vlabels
def lovasz_loss(y_true, y_pred):
y_true, y_pred = K.cast(K.squeeze(y_true, -1), 'int32'), K.cast(K.squeeze(y_pred, -1), 'float32')
 logits = y_pred
loss = lovasz_hinge(logits, y_true, per_image = True, ignore = None)
 return loss
train.apply(lambda col: len(col.unique()))<drop_column>
x_train = np.append(x_train, [np.fliplr(x)for x in x_train], axis=0)
 y_train = np.append(y_train, [np.fliplr(x)for x in y_train], axis=0)
 print(x_train.shape)
 print(y_valid.shape )
train.drop('fare', axis=1, inplace=True)
 test.drop('fare', axis=1, inplace=True )<categorify>
if Path(previous_model_name ).is_file() :
print("Using previous sucessful run's model")
 model2 = load_model(previous_model_name,custom_objects={'my_iou_metric_2': my_iou_metric,
'lovasz_loss': lovasz_loss,
'my_iou_metric': my_iou_metric})
 else:
print("Using stored trained model")
 model2 = load_model(stored_trained_model,custom_objects={'my_iou_metric_2': my_iou_metric,
'lovasz_loss': lovasz_loss,
'my_iou_metric': my_iou_metric} )
def check_features_list(features):

 if type(features)== str:
features = [features]
return features
def convert_categorical_features(df, cat_features):

 if len(cat_features)== 0:
return df
cat_features = check_features_list(cat_features)
 for feature in cat_features:
dummies = pd.get_dummies(df[feature], prefix=feature)
 df = pd.concat([df,dummies], axis=1)
 df.drop(cat_features, axis=1, inplace=True)
 return df<choose_model_class>
input_x = model2.layers[0].input
output_layer = Activation('sigmoid',name='output_activaton' )(model2.layers[-1].output)
 model1 = Model(input_x, output_layer)
 c = optimizers.adam(lr = 0.01)
 model1.compile(loss="binary_crossentropy", optimizer=c, metrics=[my_iou_metric])
lr = LogisticRegression()
cat_features = ['pclass','sex','embarked']
for feature in cat_features:
accuracy = evaluate_lr_model(train, cat_features=feature)
 print('{}: {:.2f}%'.format(feature, accuracy*100))
num_features = ['sibsp','parch']
for feature in num_features:
accuracy = evaluate_lr_model(train, num_features=feature)
 print('{}: {:.2f}%'.format(feature, accuracy*100))<compute_test_metric>
model_checkpoint = ModelCheckpoint(save_model_name,monitor='my_iou_metric',
mode = 'max', save_best_only=True, verbose=1)
 reduce_lr = ReduceLROnPlateau(monitor='my_iou_metric', mode = 'max',factor=0.5, patience=5, min_lr=0.0001, verbose=1)
 epochs = 10
batch_size = 32
history = model1.fit(x_train, y_train,
validation_data=[x_valid, y_valid],
epochs=epochs,
batch_size=batch_size,
callbacks=[ model_checkpoint,reduce_lr],
verbose=2 )
accuracy = evaluate_lr_model(train, cat_features, num_features)
 print('{}: {:.2f}%'.format(cat_features + num_features, accuracy*100))<save_to_csv>
model1 = load_model(save_model_name,custom_objects={'my_iou_metric': my_iou_metric})
 input_x = model1.layers[0].input
output_layer = model1.layers[-1].input
model = Model(input_x, output_layer)
 c = optimizers.adam(lr = 0.01)
 model.compile(loss=lovasz_loss, optimizer=c, metrics=[my_iou_metric_2])
def predict_test(model, train_c, test_c, filenamne):

 features = train_c.columns.to_list()
features.remove(target)
 model.fit(train_c[features], train_c[target])
 predictions = model.predict(test_c[features])
 submission = pd.DataFrame({"PassengerId":test["passengerid"],"Survived":predictions})
 submission.to_csv(filenamne, index=False)
 train_c = copy_convert_dataset(train, cat_features, num_features)
 test_c = copy_convert_dataset(test, cat_features, num_features, False)
 predict_test(lr, train_c, test_c, 'submission_1.csv' )<compute_test_metric>
early_stopping = EarlyStopping(monitor='val_my_iou_metric_2', mode = 'max',patience=30, verbose=1)
 model_checkpoint = ModelCheckpoint(save_model_name,monitor='val_my_iou_metric_2',
mode = 'max', save_best_only=True, verbose=1)
 reduce_lr = ReduceLROnPlateau(monitor='val_my_iou_metric_2', mode = 'max',factor=0.5, patience=5, min_lr=0.0001, verbose=1)
 epochs = 50
batch_size = 32
history = model.fit(x_train, y_train,
validation_data=[x_valid, y_valid],
epochs=epochs,
batch_size=batch_size,
callbacks=[ model_checkpoint,reduce_lr,early_stopping],
verbose=2 )
cat_features = ['pclass','sex','embarked','age_group']
accuracy = evaluate_lr_model(train, cat_features, num_features)
 print('{}: {:.2f}%'.format(cat_features + num_features, accuracy*100))<define_variables>
model = model2
titles_map = {
"Mr" : "Mr",
"Mme": "Mrs",
"Ms": "Mrs",
"Mrs" : "Mrs",
"Master" : "Master",
"Mlle": "Miss",
"Miss" : "Miss",
"Capt": "Officer",
"Col": "Officer",
"Major": "Officer",
"Dr": "Officer",
"Rev": "Officer",
"Jonkheer": "Royalty",
"Don": "Royalty",
"Sir" : "Royalty",
"Countess": "Royalty",
"Dona": "Royalty",
"Lady" : "Royalty"
}
def extract_title(name):

 return name.str.extract('([A-Za-z]+)\.', expand=False ).map(titles_map)
 train["title"] = extract_title(train['name'])
 train.pivot_table(index='title',values='survived')['survived'].plot.barh()
set_spines()<compute_test_metric>
model = load_model(save_model_name,custom_objects={'my_iou_metric_2': my_iou_metric_2,
'lovasz_loss': lovasz_loss})
cat_features = ['pclass','sex','embarked','age_group','title']
accuracy = evaluate_lr_model(train, cat_features, num_features)
 print('{}: {:.2f}%'.format(cat_features + num_features, accuracy*100))<feature_engineering>
def predict_result(model,x_test,img_size_target):
x_test_reflect = np.array([np.fliplr(x)for x in x_test])
 preds_test = model.predict(x_test ).reshape(-1, img_size_target, img_size_target)
 preds_test2_refect = model.predict(x_test_reflect ).reshape(-1, img_size_target, img_size_target)
 preds_test += np.array([ np.fliplr(x)for x in preds_test2_refect])
 return preds_test/2
test['age_group'] = group_age(test['age'])
 test["title"] = extract_title(test['name'])
 train_c = copy_convert_dataset(train, cat_features, num_features)
 test_c = copy_convert_dataset(test, cat_features, num_features, False)
 predict_test(lr, train_c, test_c, 'submission_2.csv' )<choose_model_class>
preds_train = predict_result(model,x_train,img_size_target)
 preds_valid = predict_result(model,x_valid,img_size_target )
models = [
{
'name':'Logistic regression',
'estimator':LogisticRegression() ,
'hyperparameters':{
'solver': ['newton-cg', 'lbfgs', 'liblinear']
}
},
{
'name':'Decision tree',
'estimator':DecisionTreeClassifier(random_state=1),
'hyperparameters':{
'criterion':['entropy','gini'],
'splitter':['best','random'],
'max_depth':range(4,10),
'max_features':['auto','sqrt','log2'],
'min_samples_split':[2,3,4]
}
}, {
'name':'RandomForestClassifier',
'estimator':RandomForestClassifier(n_estimators=300, random_state=1),
'hyperparameters':{
'criterion':['entropy','gini'],
'max_depth':range(4,9),
'max_features':['auto','sqrt','log2'],
'min_samples_split':[2,3,4]
}
}
]
def tune_models(models, df):
start = time()
for model in models:
print(model['name'])
 print('-' * len(model['name']))
grid = GridSearchCV(model['estimator'], param_grid=model['hyperparameters'], cv=5)
 grid.fit(df.loc[:,df.columns != target], df[target])
 model['best_score'] = grid.best_score_
model['best_params'] = grid.best_params_
model['best_model'] = grid.best_estimator_
print('Best accuracy: {:.2f}%'.format(model['best_score']*100))
print('Best parameters: {}
'.format(model['best_params']))
end = time()
print()
print('Elapsed time: {:.2f}s'.format(end - start))
return models
models = tune_models(models, train_c )<predict_on_test>
thresholds_ori = np.linspace(0.3, 0.7, 31)
 thresholds = np.log(thresholds_ori/(1-thresholds_ori))
ious = np.array([iou_metric_batch(y_valid, preds_valid > threshold)for threshold in tqdm_notebook(thresholds)])
 print(ious )
predict_test(models[2]['best_model'], train_c, test_c, 'submission_3.csv' )<set_options>

 def rle_encode(im):

 pixels = im.flatten(order = 'F')
 pixels = np.concatenate([[0], pixels, [0]])
 runs = np.where(pixels[1:] != pixels[:-1])[0] + 1
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs )
np.random.seed(0)
 warnings.filterwarnings('ignore')
<load_from_csv>
x_all_train = np.array(train_df.images.map(upsample ).tolist() ).reshape(-1, img_size_target, img_size_target, 1)
 preds_train = predict_result(model,x_all_train,img_size_target)
 preds_test = predict_result(model,x_test,img_size_target )
train = pd.read_csv('/kaggle/input/web-traffic-time-series-forecasting/train_2.csv.zip' ).fillna(0 )<compute_test_metric>
all_preds = list(np.concatenate(( preds_train,preds_test)))
 preds_df = pd.DataFrame(pd.Series(all_preds), columns = ['pred_masks'])
 preds_df.insert(0, 'orig_id', range(1,len(preds_df)+1))
all_arr = pd.merge(all_arr,preds_df,how = 'left',left_on = 'orig_id', right_on = 'orig_id')
def rmsse(train, y, y_hat):
num = np.sum(np.power(( y - y_hat), 2))
den =(1 /(len(train)- 1)) * np.sum(np.power(np.diff(train), 2))
return np.sqrt(( 1 / len(y)) * num / den)
 def smape(y_true, y_pred):
denominator =(np.abs(y_true)+ np.abs(y_pred)) / 200.0
diff = np.abs(y_true - y_pred)/ denominator
diff[denominator == 0] = 0.0
return np.nanmean(diff)
 def rmse(y_true, y_pred):
return np.sqrt(mean_squared_error(y_true, y_pred))
def ts_folds(x, n, h, val_type='exp_wind'):
assert len(x)- 2 * h > n * h, "horisont or number of folds are too large."
assert n > 0 and h > 0
l = len(x)
 folds = []
if val_type == 'exp_wind':
for i in range(n):
folds.append(( x[:l - h*(i + 1)], x[l - h*(i+1):l - h*i]))
return folds
def plot_ts(x_ts, info):
x = x_ts.values
ts_name = 'X'
fig, axes = plt.subplots(3, 2, figsize=(18, 20))
axes[0, 0].set_title(f"Original, id: {info['id']}", fontsize=16)
 axes[0, 0].plot(x, label=ts_name)
 axes[0, 0].set_ylabel('X', fontsize=16)
 axes[0, 0].set_xlabel('t', fontsize=16)
 axes[0, 0].legend()
plot_acf(x, lags=90, ax=axes[0, 1])
 sns.distplot(x, bins=100, kde=True, ax=axes[1, 0])
 axes[1, 0].set_xlabel('X', fontsize=16)
 axes[1, 0].set_title('Histogram', fontsize=16)
 x_ts.diff().plot(ax=axes[1, 1], title='1 step difference')
 axes[2, 0].set_title('Box plot(grouped by months)', fontsize=16)
 sns.boxplot(x_ts.index.month, x_ts, ax=axes[2, 0])
 axes[2, 1].plot(x_ts, label=ts_name)
 x_ts.rolling(window=7 ).mean().plot(ax=axes[2, 1], label='W=7')
 x_ts.rolling(window=30 ).mean().plot(ax=axes[2, 1], label='W=30')
 x_ts.rolling(window=60 ).mean().plot(ax=axes[2, 1], label='W=60')
 axes[2, 1].legend()
axes[2, 1].set_title('Moving average', fontsize=16)
 axes[2, 1].set_ylabel('X', fontsize=16)
 axes[2, 1].set_xlabel('t', fontsize=16)
 plt.show()
res = seasonal_decompose(x_ts, model='additive')
 fig,(ax1, ax2, ax3, ax4)= plt.subplots(4, 1, figsize=(12, 15))
fig.suptitle('additive decomposition')
 res.observed.plot(ax=ax1)
 ax1.set_ylabel('original')
 res.trend.plot(ax=ax2)
 ax2.set_ylabel('trend')
 res.seasonal.plot(ax=ax3)
 ax3.set_ylabel('seasonal')
 res.resid.plot(ax=ax4)
 ax4.set_ylabel('residuals')
 pass
def plot_prediction(train, val, y_hat, plot_len=None, title=None):
if plot_len == None:
plot_len = len(train)+ len(val)
 l = len(train)
 train_range = np.arange(l)
 val_range = np.arange(l, l + len(val))
all_range = np.arange(l + len(val))
fig = plt.figure(figsize=(14, 8))
plt.plot(train_range, train, '.-b', label='train')
 plt.plot(val_range, val, '.-g', label='val')
 plt.plot(val_range, y_hat, '.-r', label='forecast')
 plt.xlim([all_range[-plot_len:][0], all_range[-plot_len:][-1]])
 plt.legend()
plt.vlines(l, -1e6, 1e6, 'k', 'dashed')
 plt.ylim([np.min([np.min(train[-plot_len:]), np.min(val)])-1, np.max([np.max(train[-plot_len:]), np.max(val)])+10])
 plt.title(title)
 plt.show()
class TransformTS() :
def __init__(self, tr_type, handle_missing=None):
self.tr_type = tr_type
self.handle_missing = handle_missing
def transform(self, x):
if self.tr_type == 'ln_p1':
y = np.log(x + 1)
 elif self.tr_type == 'ln':
y = np.log(x)
 elif self.tr_type == 'sqrt':
y = np.sqrt(x)
 elif self.tr_type == 'boxcox':
y, lb = stats.boxcox(x)
 self.lb = lb
else:
raise ValueError
return y
def reverse(self, x):
if self.tr_type == 'ln_p1':
y = np.exp(x)- 1
elif self.tr_type == 'ln':
y = np.exp(x)
 elif self.tr_type == 'sqrt':
y = np.power(x, 2)
 elif self.tr_type == 'boxcox':
y = scipy.special.inv_boxcox(x, self.lb)
 else:
raise ValueError
return y 
<compute_train_metric>
all_arr_saved = all_arr.copy()
HORIZONT = 64
def prophet_fc(x, h=HORIZONT):
m = Prophet()
m.fit(x)
 future = m.make_future_dataframe(periods=h)
 forecast = m.predict(future)
 y_hat = forecast['yhat'][-h:]
return y_hat
def naive(x, h=HORIZONT):
return np.ones(h)* np.mean(x[-HORIZONT:])
 def naive2(x, h=HORIZONT):
return np.ones(h)* x[-1]
def arima_mdl(x, order, h=HORIZONT):
p, d, q = order
l = len(x)
 mod = sm.tsa.statespace.SARIMAX(x, order=(p, d, q), trend='c' ).fit(disp=False)
 y_hat = mod.forecast(steps=HORIZONT)
 return y_hat
def agg_weekly(x, window=64, h=HORIZONT, avg_type='mean'):
xs = x[-window:]
medians = [xs[xs.index.weekday == i].median() for i in range(7)]
means = [xs[xs.index.weekday == i].mean() for i in range(7)]
d = x.index[-1].weekday()
y_hat = np.zeros(HORIZONT)
 for n in range(HORIZONT):
if d == 6:
d = 0
else:
d = d + 1
if avg_type == 'median':
y_hat[n] = medians[d]
elif avg_type == 'mean':
y_hat[n] = means[d]
else:
raise ValueError
return y_hat
def exp_smoothing(x_train, config, h=HORIZONT):
t,d,s,p,b,r = config
model = ExponentialSmoothing(x_train, trend=t, damped=d, seasonal=s, seasonal_periods=p)
 model_fit = model.fit(optimized=True, use_boxcox=b, remove_bias=r)
 yhat = model_fit.forecast(h)
 return yhat 
<define_search_space>
preds_test = np.array(all_arr.loc[all_arr['img_id']>3999,'pred_masks'].tolist())
 preds_train = np.array(all_arr.loc[all_arr['img_id']<=3999,'pred_masks'].tolist())
 y_train = np.array(train_df.masks.map(upsample ).tolist() ).reshape(-1, img_size_target, img_size_target, 1)
t_params = ['add', 'mul', None]
d_params = [True, False]
s_params = ['add', 'mul', None]
p_params = [5, 7, 12, 14, 30]
b_params = [True, False]
r_params = [True, False]
cfg_list = []
for t in t_params:
for d in d_params:
for s in s_params:
for p in p_params:
for b in b_params:
for r in r_params:
if d is True and t is None:
continue
cfg = [t, d, s, p, b, r]
cfg_list.append(cfg)
<compute_test_metric>
t1 = time.time()
pred_dict = {idx: rle_encode(np.round(downsample(preds_test[i])> threshold_best)) for i, idx in enumerate(tqdm_notebook(test_df.index.values)) }
t2 = time.time()
print(f"Usedtime = {t2-t1} s" )
def eval_models(train, y_val, m_list):
models = []
if 'naive1' in m_list:
y_hat = naive(train.values, h=HORIZONT)
 y_hat[y_hat < 0] = 0
models.append({
'name': 'naive1',
'y_pred': y_hat,
'smape': smape(y_val, y_hat),
'rmse': np.sqrt(mean_squared_error(y_hat, y_val)) ,
'model': None} 
)
 if 'naive2' in m_list:
y_hat = naive2(train.values, h=HORIZONT)
 y_hat[y_hat < 0] = 0
models.append({
'name': 'naive2',
'y_pred': y_hat,
'smape': smape(y_val, y_hat),
'rmse': np.sqrt(mean_squared_error(y_hat, y_val)) ,
'model': None} 
)
 if 'prophet' in m_list:
x_df = pd.DataFrame({'ds': train.index, 'y': x_train.values})
 y_hat = prophet_fc(x_df, h=HORIZONT)
 y_hat[y_hat < 0] = 0
models.append({
'name': 'prophet',
'y_pred': y_hat,
'smape': smape(y_val, y_hat),
'rmse': np.sqrt(mean_squared_error(y_hat, y_val)) ,
'model': None} 
)
 for p in range(30):
for d in [0, 1]:
for q in range(4):
m_name = f'arima_{p}_{d}_{q}'
if m_name in m_list:
try:
y_hat = arima_mdl(train,(p, d, q), h=HORIZONT)
 conv = True
except:
y_hat = naive(train.values, h=HORIZONT)
 conv = False
y_hat[y_hat < 0] = 0
models.append({
'name': m_name,
'y_pred': y_hat,
'smape': smape(y_val, y_hat),
'rmse': np.sqrt(mean_squared_error(y_hat, y_val)) ,
'model': None,
'conv': conv,
})
 winds = [7, 32, 64, 96]
for w in winds:
if f'weekly_mean_w-{w}' in m_list:
y_hat = agg_weekly(train, window=w, h=HORIZONT, avg_type='mean')
 y_hat[y_hat < 0] = 0
models.append({
'name': f'weekly_mean_w-{w}',
'y_pred': y_hat,
'smape': smape(y_val, y_hat),
'rmse': np.sqrt(mean_squared_error(y_hat, y_val)) ,
'model': None} 
)
 winds = np.arange(1, 13)* 7
for w in winds:
if f'weekly_median_w-{w}' in m_list:
y_hat = agg_weekly(train, window=w, h=HORIZONT, avg_type='median')
 y_hat[y_hat < 0] = 0
models.append({
'name': f'weekly_median_w-{w}',
'y_pred': y_hat,
'smape': smape(y_val, y_hat),
'rmse': np.sqrt(mean_squared_error(y_hat, y_val)) ,
'model': None} 
)
 for cfg in cfg_list:
if f"ets_{cfg}" in m_list:
train = train + 1
y_hat = exp_smoothing(train, cfg, h=HORIZONT)
 if y_hat.isnull().values.any() or np.isinf(y_hat.values ).any() :
y_hat = naive(train.values, h=HORIZONT)
 conv = False
else:
conv = True
y_hat = y_hat - 1
y_hat[y_hat < 0] = 0
models.append({
'name': f'ets_{cfg}',
'y_pred': y_hat,
'smape': smape(y_val, y_hat),
'rmse': np.sqrt(mean_squared_error(y_hat, y_val)) ,
'model': None,
'conv': conv,} 
)
 return models 
<define_variables>
sub = pd.DataFrame.from_dict(pred_dict,orient='index')
 sub.index.names = ['id']
sub.columns = ['rle_mask']
sub.to_csv(submission_file )
<feature_engineering><EOS> 
t_finish = time.time()
print(f"Kernel run time = {(t_finish-t_start)/3600} hours" )