Kaggle competition: Porto Seguro’s Safe Driver Prediction
<SOS> metric: NormalizedGini Kaggle data source: porto-seguro’s-safe-driver-prediction<define_search_model>
warnings.filterwarnings('ignore' )
def create_downsample(channels, inputs):
x = keras.layers.BatchNormalization(momentum=0.99999 )(inputs)
 x = keras.layers.LeakyReLU(0 )(x)
 x = keras.layers.Conv2D(channels, 1, padding='same', use_bias=False )(x)
 x = keras.layers.MaxPool2D(( 2, 2))(x)
 return x
def create_resblock(channels, inputs):
x = keras.layers.BatchNormalization(momentum=0.9999 )(inputs)
 x = keras.layers.LeakyReLU(0 )(x)
 x = keras.layers.Conv2D(channels, 3, padding='same', use_bias=False )(x)
 x = keras.layers.BatchNormalization(momentum=0.9999 )(x)
 x = keras.layers.LeakyReLU(0 )(x)
 x = keras.layers.Conv2D(channels, 3, padding='same', use_bias=False )(x)
 return keras.layers.add([x, inputs])
 def create_network(input_size, channels, n_blocks=2, depth=4):
inputs = keras.Input(shape=(input_size, input_size, 1))
x = keras.layers.Conv2D(channels, 3, padding='same', use_bias=False )(inputs)
 for d in range(depth):
channels = channels * 2
x = create_downsample(channels, x)
 for b in range(n_blocks):
x = create_resblock(channels, x)
 x = keras.layers.BatchNormalization(momentum=0.9999 )(x)
 x = keras.layers.LeakyReLU(0 )(x)
 x = keras.layers.Conv2D(1, 1, activation='sigmoid' )(x)
 outputs = keras.layers.UpSampling2D(2**depth )(x)
 model = keras.Model(inputs=inputs, outputs=outputs)
 return model<compute_test_metric>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )
def iou_loss(y_true, y_pred):
y_true = tf.reshape(y_true, [-1])
 y_pred = tf.reshape(y_pred, [-1])
 intersection = tf.reduce_sum(y_true * y_pred)
 score =(intersection + 1.) /(tf.reduce_sum(y_true)+ tf.reduce_sum(y_pred)- intersection + 1.)
return 1 - score
def iou_bce_loss(y_true, y_pred):
return 0.4 * keras.losses.binary_crossentropy(y_true, y_pred)+ 0.6 * iou_loss(y_true, y_pred)
 def mean_iou(y_true, y_pred):
y_pred = tf.round(y_pred)
 intersect = tf.reduce_sum(y_true * y_pred, axis=[1, 2, 3])
 union = tf.reduce_sum(y_true, axis=[1, 2, 3])+ tf.reduce_sum(y_pred, axis=[1, 2, 3])
 smooth = tf.ones(tf.shape(intersect))
return tf.reduce_mean(( intersect + smooth)/(union - intersect + smooth))
model = create_network(input_size=256, channels=32, n_blocks=2, depth=4)
 model.compile(optimizer='adam',
loss=iou_bce_loss,
metrics=['accuracy', mean_iou])
 def cosine_annealing(x):
lr = 0.01
epochs = 20
return lr*(np.cos(np.pi*x/epochs)+1.) /2
learning_rate = tf.keras.callbacks.LearningRateScheduler(cosine_annealing)
 folder = '.. /input/stage_1_train_images'
train_gen = generator(folder, train_filenames, pneumonia_locations, batch_size=16, image_size=256, shuffle=True, augment=True, predict=False)
 valid_gen = generator(folder, valid_filenames, pneumonia_locations, batch_size=16, image_size=256, shuffle=False, predict=False)
 history = model.fit_generator(train_gen, validation_data=valid_gen, callbacks=[learning_rate], epochs=20, shuffle=True )<predict_on_test>
id_test = test['id'].values
target_train = train['target'].values
train = train.drop(['target','id'], axis = 1)
 test = test.drop(['id'], axis = 1)
 col_to_drop = train.columns[train.columns.str.startswith('ps_calc_')]
train = train.drop(col_to_drop, axis=1)
 test = test.drop(col_to_drop, axis=1)
 train = train.replace(-1, np.nan)
 test = test.replace(-1, np.nan)
 cat_features = [a for a in train.columns if a.endswith('cat')]
for column in cat_features:
temp = pd.get_dummies(pd.Series(train[column]))
train = pd.concat([train,temp],axis=1)
 train = train.drop([column],axis=1)
 for column in cat_features:
temp = pd.get_dummies(pd.Series(test[column]))
test = pd.concat([test,temp],axis=1)
 test = test.drop([column],axis=1)
 print(train.values.shape, test.values.shape )
folder = '.. /input/stage_1_test_images'
test_filenames = os.listdir(folder)
 print('n test samples:', len(test_filenames))
test_gen = generator(folder, test_filenames, None, batch_size=20, image_size=256, shuffle=False, predict=True)
 submission_dict = {}
for imgs, filenames in test_gen:
preds = model.predict(imgs)
 for pred, filename in zip(preds, filenames):
pred = resize(pred,(1024, 1024), mode='reflect')
 comp = pred[:, :, 0] > 0.5
comp = measure.label(comp)
 predictionString = ''
for region in measure.regionprops(comp):
y, x, y2, x2 = region.bbox
height = y2 - y
width = x2 - x
conf = np.mean(pred[y:y+height, x:x+width])
 predictionString += str(conf)+ ' ' + str(x)+ ' ' + str(y)+ ' ' + str(width)+ ' ' + str(height)+ ' '
filename = filename.split('.')[0]
submission_dict[filename] = predictionString
if len(submission_dict)>= len(test_filenames):
break
sub = pd.DataFrame.from_dict(submission_dict,orient='index')
 sub.index.names = ['patientId']
sub.columns = ['PredictionString']
sub.to_csv('submission.csv' )<set_options>
class Ensemble(object):
def __init__(self, mode, n_splits, stacker_2, stacker_1, base_models):
self.mode = mode
self.n_splits = n_splits
self.stacker_2 = stacker_2
self.stacker_1 = stacker_1
self.base_models = base_models
def fit_predict(self, X, y, T):
X = np.array(X)
 y = np.array(y)
 T = np.array(T)
 folds = list(StratifiedKFold(n_splits=self.n_splits, shuffle=True,
random_state=2016 ).split(X, y))
OOF_columns = []
S_train = np.zeros(( X.shape[0], len(self.base_models)))
 S_test = np.zeros(( T.shape[0], len(self.base_models)))
 for i, clf in enumerate(self.base_models):
S_test_i = np.zeros(( T.shape[0], self.n_splits))
for j,(train_idx, test_idx)in enumerate(folds):
X_train = X[train_idx]
y_train = y[train_idx]
X_holdout = X[test_idx]
print("Fit %s_%d fold %d" %(str(clf ).split("(")[0], i+1, j+1))
clf.fit(X_train, y_train)
 S_train[test_idx, i] = clf.predict_proba(X_holdout)[:,1]
S_test_i[:, j] = clf.predict_proba(T)[:,1]
S_test[:, i] = S_test_i.mean(axis=1)
 print(" Base model_%d score: %.5f
" %(i+1, roc_auc_score(y, S_train[:,i])))
 OOF_columns.append('Base model_'+str(i+1))
OOF_S_train = pd.DataFrame(S_train, columns = OOF_columns)
 print('
')
 print('Correlation between out-of-fold predictions from Base models:')
 print('
')
 print(OOF_S_train.corr())
 print('
')
 if self.mode==1:
folds_2 = list(StratifiedKFold(n_splits=self.n_splits, shuffle=True,
random_state=2016 ).split(S_train, y))
OOF_columns = []
S_train_2 = np.zeros(( S_train.shape[0], len(self.stacker_1)))
 S_test_2 = np.zeros(( S_test.shape[0], len(self.stacker_1)))
 for i, clf in enumerate(self.stacker_1):
S_test_i_2 = np.zeros(( S_test.shape[0], self.n_splits))
for j,(train_idx, test_idx)in enumerate(folds_2):
X_train_2 = S_train[train_idx]
y_train_2 = y[train_idx]
X_holdout_2 = S_train[test_idx]
print("Fit %s_%d fold %d" %(str(clf ).split("(")[0], i+1, j+1))
clf.fit(X_train_2, y_train_2)
 S_train_2[test_idx, i] = clf.predict_proba(X_holdout_2)[:,1]
S_test_i_2[:, j] = clf.predict_proba(S_test)[:,1]
S_test_2[:, i] = S_test_i_2.mean(axis=1)
 print(" 1st level model_%d score: %.5f
"%(i+1,
roc_auc_score(y, S_train_2.mean(axis=1))))
OOF_columns.append('1st level model_'+str(i+1))
OOF_S_train = pd.DataFrame(S_train_2, columns = OOF_columns)
 print('
')
 print('Correlation between out-of-fold predictions from 1st level models:')
 print('
')
 print(OOF_S_train.corr())
 print('
')
 if self.mode==2:
WOC_columns = []
S_train_2 = np.zeros(( S_train.shape[0], len(self.stacker_1)))
 S_test_2 = np.zeros(( S_test.shape[0], len(self.stacker_1)))
 for i, clf in enumerate(self.stacker_1):
S_train_i_2= np.zeros(( S_train.shape[0], S_train.shape[1]))
S_test_i_2 = np.zeros(( S_test.shape[0], S_train.shape[1]))
for j in range(S_train.shape[1]):
S_tr = S_train[:,np.arange(S_train.shape[1])!=j]
S_te = S_test[:,np.arange(S_test.shape[1])!=j]
print("Fit %s_%d subset %d" %(str(clf ).split("(")[0], i+1, j+1))
clf.fit(S_tr, y)
 S_train_i_2[:, j] = clf.predict_proba(S_tr)[:,1]
S_test_i_2[:, j] = clf.predict_proba(S_te)[:,1]
S_train_2[:, i] = S_train_i_2.mean(axis=1)
 S_test_2[:, i] = S_test_i_2.mean(axis=1)
 print(" 1st level model_%d score: %.5f
"%(i+1,
roc_auc_score(y, S_train_2.mean(axis=1))))
WOC_columns.append('1st level model_'+str(i+1))
WOC_S_train = pd.DataFrame(S_train_2, columns = WOC_columns)
 print('
')
 print('Correlation between without-one-column predictions from 1st level models:')
 print('
')
 print(WOC_S_train.corr())
 print('
')
 try:
num_models = len(self.stacker_2)
 if self.stacker_2==(et_model):
num_models=1
except TypeError:
num_models = len([self.stacker_2])
 if num_models==1:
print("Fit %s for final
" %(str(self.stacker_2 ).split("(")[0]))
self.stacker_2.fit(S_train_2, y)
 stack_res = self.stacker_2.predict_proba(S_test_2)[:,1]
stack_score = self.stacker_2.predict_proba(S_train_2)[:,1]
print("2nd level model final score: %.5f" %(roc_auc_score(y, stack_score)))
 else:
F_columns = []
stack_score = np.zeros(( S_train_2.shape[0], len(self.stacker_2)))
 res = np.zeros(( S_test_2.shape[0], len(self.stacker_2)))
 for i, clf in enumerate(self.stacker_2):
print("Fit %s_%d" %(str(clf ).split("(")[0], i+1))
clf.fit(S_train_2, y)
 stack_score[:, i] = clf.predict_proba(S_train_2)[:,1]
print(" 2nd level model_%d score: %.5f
"%(i+1,roc_auc_score(y, stack_score[:, i])))
 res[:, i] = clf.predict_proba(S_test_2)[:,1]
F_columns.append('2nd level model_'+str(i+1))
F_S_train = pd.DataFrame(stack_score, columns = F_columns)
 print('
')
 print('Correlation between final predictions from 2nd level models:')
 print('
')
 print(F_S_train.corr())
 print('
')
 stack_res = res.mean(axis=1)
 print("2nd level models final score: %.5f" %(roc_auc_score(y, stack_score.mean(axis=1))))
return stack_res
warnings.filterwarnings('ignore' )<load_from_csv>
lgb_params_1 = {
'learning_rate': 0.02,
'n_estimators': 750,
'subsample': 0.8,
'subsample_freq': 10,
'colsample_bytree': 0.8,
'max_bin': 10,
'min_child_samples': 500,
'seed': 99
}
lgb_params_2 = {
'learning_rate': 0.02,
'n_estimators': 1200,
'subsample': 0.7,
'subsample_freq': 2,
'colsample_bytree': 0.3,
'num_leaves': 16,
'seed': 99
}
lgb_params_3 = {
'learning_rate': 0.02,
'n_estimators': 475,
'subsample': 0.4,
'subsample_freq': 1,
'colsample_bytree': 0.9,
'num_leaves': 28,
'max_bin': 10,
'min_child_samples': 700,
'seed': 99
}
train = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 test = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv')
 total = [train, test]<sort_values>
lgb_model_1 = LGBMClassifier(**lgb_params_1)
 lgb_model_2 = LGBMClassifier(**lgb_params_2)
 lgb_model_3 = LGBMClassifier(**lgb_params_3 )
train.isnull().sum().sort_values(ascending=False)[train.isnull().sum().sort_values(ascending=False)> 0] / train.shape[0] * 100<sort_values>
log_model = LogisticRegression()
et_model = ExtraTreesClassifier(n_estimators=100, max_depth=6, min_samples_split=10, random_state=10)
 mlp_model = MLPClassifier(max_iter=7, random_state=42 )
test.isnull().sum().sort_values(ascending=False)[test.isnull().sum().sort_values(ascending=False)> 0] / test.shape[0] * 100<data_type_conversions>
stack = Ensemble(mode=2,
n_splits=3,
stacker_2 =(log_model, et_model),
stacker_1 =(log_model, et_model, mlp_model),
base_models =(lgb_model_1, lgb_model_2, lgb_model_3))
y_pred = stack.fit_predict(train, target_train, test )
<feature_engineering><EOS> 
sub = pd.DataFrame()
sub['id'] = id_test
sub['target'] = y_pred
sub.to_csv('2-level_stacked_1.csv', index=False )