Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<choose_model_class>
sns.set(style="whitegrid")
 %matplotlib inline
warnings.simplefilter(action='ignore')
 training_data = pd.read_csv('.. /input/train.csv')
 testing_data = pd.read_csv('.. /input/test.csv' )
attn_layer = Conv2D(64, kernel_size =(1,1), padding = 'same', activation = 'relu' )(Dropout(0.5 )(bn_features))
attn_layer = Conv2D(16, kernel_size =(1,1), padding = 'same', activation = 'relu' )(attn_layer)
 attn_layer = Conv2D(8, kernel_size =(1,1), padding = 'same', activation = 'relu' )(attn_layer)
 attn_layer = Conv2D(1,
kernel_size =(1,1),
padding = 'valid',
activation = 'sigmoid' )(attn_layer)
 up_c2_w = np.ones(( 1, 1, 1, pt_depth))
up_c2 = Conv2D(pt_depth, kernel_size =(1,1), padding = 'same',
activation = 'linear', use_bias = False, weights = [up_c2_w])
 up_c2.trainable = False
attn_layer = up_c2(attn_layer)
 mask_features = multiply([attn_layer, bn_features])
 gap_features = GlobalAveragePooling2D()(mask_features)
 gap_mask = GlobalAveragePooling2D()(attn_layer)
 gap = Lambda(lambda x: x[0]/x[1], name = 'RescaleGAP' )([gap_features, gap_mask])
 gap_dr = Dropout(0.25 )(gap)
 dr_steps = Dropout(0.25 )(Dense(128, activation = 'relu' )(gap_dr))
out_layer = Dense(5, activation = 'softmax' )(dr_steps)
 retina_model = Model(inputs = [in_lay], outputs = [out_layer] )<choose_model_class>
sex_dict = {'male':0, 'female':1}
EarlyStopping, ReduceLROnPlateau,CSVLogger)
 epochs = 30; batch_size = 16
checkpoint = ModelCheckpoint('.. /working/model_.h5', monitor='val_loss', verbose=1,
save_best_only=True, mode='min', save_weights_only = True)
 reduceLROnPlat = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=4,
verbose=1, mode='auto', epsilon=0.0001)
 early = EarlyStopping(monitor="val_loss",
mode="min",
patience=9)
 csv_logger = CSVLogger(filename='.. /working/training_log.csv',
separator=',',
append=True)
 train_generator = My_Generator(train_x, train_y, 128, is_train=True)
 train_mixup = My_Generator(train_x, train_y, batch_size, is_train=True, mix=False, augment=True)
 valid_generator = My_Generator(valid_x, valid_y, batch_size, is_train=False )<compute_test_metric>
training_data['Sex'] = training_data['Sex'].map(sex_dict)
 training_data.head()
def kappa_loss(y_true, y_pred, y_pow=2, eps=1e-12, N=5, bsize=32, name='kappa'):
with tf.name_scope(name):
y_true = tf.to_float(y_true)
 repeat_op = tf.to_float(tf.tile(tf.reshape(tf.range(0, N), [N, 1]), [1, N]))
repeat_op_sq = tf.square(( repeat_op - tf.transpose(repeat_op)))
 weights = repeat_op_sq / tf.to_float(( N - 1)** 2)
 pred_ = y_pred ** y_pow
try:
pred_norm = pred_ /(eps + tf.reshape(tf.reduce_sum(pred_, 1), [-1, 1]))
except Exception:
pred_norm = pred_ /(eps + tf.reshape(tf.reduce_sum(pred_, 1), [bsize, 1]))
hist_rater_a = tf.reduce_sum(pred_norm, 0)
 hist_rater_b = tf.reduce_sum(y_true, 0)
 conf_mat = tf.matmul(tf.transpose(pred_norm), y_true)
 nom = tf.reduce_sum(weights * conf_mat)
 denom = tf.reduce_sum(weights * tf.matmul(
 tf.reshape(hist_rater_a, [N, 1]), tf.reshape(hist_rater_b, [1, N])) /
tf.to_float(bsize))
return nom*0.5 /(denom + eps)+ categorical_crossentropy(y_true, y_pred)*0.5<predict_on_test>
testing_data['Sex'] = testing_data['Sex'].map(sex_dict)
 testing_data.head()
class QWKEvaluation(Callback):
def __init__(self, validation_data=() , batch_size=64, interval=1):
super(Callback, self ).__init__()
self.interval = interval
self.batch_size = batch_size
self.valid_generator, self.y_val = validation_data
self.history = []
def on_epoch_end(self, epoch, logs={}):
if epoch % self.interval == 0:
y_pred = self.model.predict_generator(generator=self.valid_generator,
steps=np.ceil(float(len(self.y_val)) / float(self.batch_size)) ,
workers=1, use_multiprocessing=False,
verbose=1)
 def flatten(y):
return np.argmax(y, axis=1 ).reshape(-1)
 score = cohen_kappa_score(flatten(self.y_val),
flatten(y_pred),
labels=[0,1,2,3,4],
weights='quadratic')
 print("
epoch: %d - QWK_score: %.6f
" %(epoch+1, score))
self.history.append(score)
 if score >= max(self.history):
print('saving checkpoint: ', score)
 self.model.save('.. /working/model_bestqwk.h5')
 qwk = QWKEvaluation(validation_data=(valid_generator, valid_y),
batch_size=batch_size, interval=1 )<train_model>
imp = Imputer(missing_values = 'NaN', strategy = 'mean', axis = 0)
 training_data[["Age"]] = imp.fit_transform(training_data[["Age"]] ).ravel()
testing_data[["Age"]] = imp.fit_transform(testing_data[["Age"]] ).ravel()
testing_data[["Fare"]] = imp.fit_transform(testing_data[["Fare"]] ).ravel()
training_data[["Embarked"]] = training_data[["Embarked"]].fillna(method = 'ffill')
 testing_data[["Embarked"]] = testing_data[["Embarked"]].fillna(method = 'ffill' )
for layer in retina_model.layers:
layer.trainable = False
for i in range(-3,0):
retina_model.layers[i].trainable = True
retina_model.compile(
 loss='categorical_crossentropy',
optimizer=Adam(1e-3))
retina_model.fit_generator(
 train_generator,
steps_per_epoch=np.ceil(float(len(train_y)) / float(128)) ,
epochs=2,
workers=WORKERS, use_multiprocessing=True,
verbose=1,
callbacks=[qwk] )<train_model>
training_data.isnull().sum(axis=0 )
for layer in retina_model.layers:
layer.trainable = True
callbacks_list = [checkpoint, csv_logger, reduceLROnPlat, early, qwk]
retina_model.compile(
 loss=kappa_loss,
optimizer=Adam(lr=1e-4))
retina_model.fit_generator(
 train_mixup,
steps_per_epoch=np.ceil(float(len(train_x)) / float(batch_size)) ,
validation_data=valid_generator,
validation_steps=np.ceil(float(len(valid_x)) / float(batch_size)) ,
epochs=epochs,
verbose=1,
workers=1, use_multiprocessing=False,
callbacks=callbacks_list )<train_model>
testing_data.isnull().sum(axis=0 )
layer_names = []
for layer in outputs:
layer_names.append(layer.name.split("/")[0])
 print("Layers going to be used for visualization: ")
 print(layer_names )<define_variables>
le = LabelEncoder()
training_data[["Ticket"]] = le.fit_transform(training_data[["Ticket"]] ).ravel()
training_data[["Embarked"]] = le.fit_transform(training_data[["Embarked"]] ).ravel()
testing_data[["Ticket"]] = le.fit_transform(testing_data[["Ticket"]])
 testing_data[["Embarked"]] = le.fit_transform(testing_data[["Embarked"]] )
def show_random_sample(idx):

 img_id = df_train.iloc[idx]['id_code']
img_path = '.. /input/aptos2019-blindness-detection/train_images/'+str(img_id)+'.png'
sample_image = cv2.imread(img_path)
 sample_image = cv2.cvtColor(sample_image, cv2.COLOR_BGR2RGB)
 sample_image = cv2.resize(sample_image,(256, 256))
sample_label = df_train.iloc[idx]["diagnosis"]
sample_image_processed = np.expand_dims(sample_image, axis=0)
 sample_image_processed = preprocess_input(sample_image_processed)
 activations = vis_model.predict(sample_image_processed)
 pred_label = np.argmax(retina_model.predict(sample_image_processed), axis=-1)[0]
sample_activation = activations[1][0,:,:,8]
sample_activation-=sample_activation.mean()
sample_activation/=sample_activation.std()
sample_activation *=255
sample_activation = np.clip(sample_activation, 0, 255 ).astype(np.uint8)
 heatmap = get_CAM(sample_image_processed, pred_label)
 heatmap = cv2.resize(heatmap,(sample_image.shape[0], sample_image.shape[1]))
heatmap = heatmap *255
heatmap = np.clip(heatmap, 0, 255 ).astype(np.uint8)
 heatmap = cv2.applyColorMap(heatmap, cv2.COLORMAP_JET)
 super_imposed_image = heatmap * 0.5 + sample_image
super_imposed_image = np.clip(super_imposed_image, 0,255 ).astype(np.uint8)
 f,ax = plt.subplots(2,2, figsize=(10,8))
ax[0,0].imshow(sample_image)
 ax[0,0].set_title(f"True label: {sample_label}
Predicted label: {pred_label}")
 ax[0,0].axis('off')
 ax[0,1].imshow(sample_activation)
 ax[0,1].set_title("Random feature map")
 ax[0,1].axis('off')
 ax[1,0].imshow(heatmap)
 ax[1,0].set_title("Class Activation Map")
 ax[1,0].axis('off')
 ax[1,1].imshow(super_imposed_image)
 ax[1,1].set_title("Activation map superimposed")
 ax[1,1].axis('off')
 plt.show()
return activations<load_from_csv>
train_data = training_data.drop(['PassengerId','Name','Cabin','Survived'],axis = 1)
 survived = training_data['Survived']
test_data = testing_data.drop(['PassengerId','Name','Cabin'], axis = 1 )
submit = pd.read_csv('.. /input/aptos2019-blindness-detection/sample_submission.csv')
 retina_model.load_weights('.. /working/model_bestqwk.h5')
 predicted = []<predict_on_test>
X_train, X_test, y_train, y_test = train_test_split(train_data, survived, test_size=0.2, random_state=42 )
for i, name in tqdm(enumerate(submit['id_code'])) :
path = os.path.join('.. /input/aptos2019-blindness-detection/test_images/', name+'.png')
 image = cv2.imread(path)
 image = cv2.resize(image,(SIZE, SIZE))
score_predict = retina_model.predict(( image[np.newaxis])/255)
 label_predict = np.argmax(score_predict)
 predicted.append(str(label_predict))<save_to_csv>
lgbm = lgb.LGBMClassifier(max_depth = 8,
num_leaves=90,
lambda_l1 = 0.1,
lambda_l2 = 0.01,
learning_rate = 0.01,
max_bin= 350,
n_estimators = 600,
reg_alpha = 1.6,
colsample_bytree = 0.9,
subsample = 0.9,
n_jobs = 6)
 lgbm.fit(X_train,y_train)
 pred = lgbm.predict(test_data )
<define_variables><EOS> 
submission = pd.DataFrame({"PassengerId": testing_data["PassengerId"],"Survived": pred})
 submission.to_csv('submission.csv',index= False )