Kaggle competition: Titanic - Machine Learning from Disaster
print(f'Numpy version : {np.__version__}')
 print(f'Tensorflow version : {tf.__version__}')
 print(f'Tensorflow Addons version : {tfa.__version__}')
 print(f'EfficientNet(library)version : {efficientnet.__version__}')
 print(f'Matplotlib version : {matplotlib.__version__}')
 print(f'Scipy version : {scipy.__version__}')
 print(f'Pandas version : {pd.__version__}')
 print(f'Scikit-Learn version : {sklearn.__version__}' )<define_variables>
%matplotlib inline
PRE_TRAINING_TIME_START = datetime.now()
AUTO = tf.data.experimental.AUTOTUNE
SEED = 42
os.environ['PYTHONHASHSEED']=str(SEED)
 random.seed(SEED)
 np.random.seed(SEED)
 os.environ['TF_DETERMINISTIC_OPS']=str(SEED)
 tf.random.set_seed(SEED)
<set_options>
df_train=pd.read_csv('.. /input/titanic/train.csv')
 df_test=pd.read_csv('.. /input/titanic/test.csv' )
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )<define_variables>
ntrain = df_train.shape[0]
ntest = df_test.shape[0]
y_train = df_train['Survived'].values
passId = df_test['PassengerId']
data = pd.concat(( df_train, df_test))
print("data size is: {}".format(data.shape))
IMAGE_SIZE =(512, 512)
 GCS_TRAIN_PATHS = [
KaggleDatasets().get_gcs_path('tfrecords'),
KaggleDatasets().get_gcs_path('tfrecords-2')
 ]
TRAINING_FILENAMES = []
for i in GCS_TRAIN_PATHS:
TRAINING_FILENAMES.append(tf.io.gfile.glob(i + '/*.tfrecords'))
TRAINING_FILENAMES = list(itertools.chain.from_iterable(TRAINING_FILENAMES))
GCS_TEST_PATH = KaggleDatasets().get_gcs_path('tfrecords-3')
 TEST_FILENAMES = tf.io.gfile.glob(GCS_TEST_PATH + '/*.tfrecords')
 print(len(TRAINING_FILENAMES))
print(len(TEST_FILENAMES))<define_variables>
print("TRAIN DATA:")
 df_train.isnull().sum()
EPOCHS = 12
DO_AUG = True
BATCH_SIZE = 256
current_epoch = 0
chance = 0
NUM_TRAINING_IMAGES = 105390
NUM_TEST_IMAGES = 12186
STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE<define_variables>
print("TEST DATA:")
 df_test.isnull().sum()
CLASSES = [str(c ).zfill(2)for c in range(0, 42)]<categorify>
data['Last_Name'] = data['Name'].apply(lambda x: str.split(x, ",")[0])
 data['Fare'].fillna(data['Fare'].mean() , inplace=True)
 default_survival_chance = 0.5
data['Family_Survival'] = default_survival_chance
for grp, grp_df in data[['Survived','Name', 'Last_Name', 'Fare', 'Ticket', 'PassengerId',
'SibSp', 'Parch', 'Age', 'Cabin']].groupby(['Last_Name', 'Fare']):
if(len(grp_df)!= 1):
for ind, row in grp_df.iterrows() :
smax = grp_df.drop(ind)['Survived'].max()
smin = grp_df.drop(ind)['Survived'].min()
passID = row['PassengerId']
if(smax == 1.0):
data.loc[data['PassengerId'] == passID, 'Family_Survival'] = 1
elif(smin == 0.0):
data.loc[data['PassengerId'] == passID, 'Family_Survival'] = 0
print("Number of passengers with family survival information:",
data.loc[data['Family_Survival']!=0.5].shape[0] )
def decode_image(image_data):
image = tf.image.decode_jpeg(image_data, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.reshape(image, [*IMAGE_SIZE, 3])
 return image
def read_labeled_tfrecord(example):
LABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"words": tf.io.FixedLenFeature([6633], tf.float32),
"label": tf.io.FixedLenFeature([], tf.int64),
}
example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 words = example['words']
label = tf.cast(example['label'], tf.int32)
 return(( image, words), label)
 def read_unlabeled_tfrecord(example):
UNLABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"words": tf.io.FixedLenFeature([6633], tf.float32),
"filename": tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 words = example['words']
filename = example['filename']
return(( image, words), filename)
 def load_dataset(filenames, labeled=True, ordered=False):
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)
 dataset = dataset.with_options(ignore_order)
 dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls=AUTO)
 return dataset
def get_training_dataset(do_aug=True):
dataset = load_dataset(TRAINING_FILENAMES, labeled=True)
 if do_aug:
dataset = dataset.map(image_augmentation, num_parallel_calls=AUTO)
 dataset = dataset.repeat()
dataset = dataset.shuffle(2048)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def get_test_dataset(ordered=False, tta=None):
dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)
 if tta == 0:
dataset = dataset.map(image_augmentation_tta_0, num_parallel_calls=AUTO)
 elif tta == 1:
dataset = dataset.map(image_augmentation_tta_1, num_parallel_calls=AUTO)
 elif tta == 2:
dataset = dataset.map(image_augmentation_tta_2, num_parallel_calls=AUTO)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset<normalization>
for _, grp_df in data.groupby('Ticket'):
if(len(grp_df)!= 1):
for ind, row in grp_df.iterrows() :
if(row['Family_Survival'] == 0)|(row['Family_Survival']== 0.5):
smax = grp_df.drop(ind)['Survived'].max()
smin = grp_df.drop(ind)['Survived'].min()
passID = row['PassengerId']
if(smax == 1.0):
data.loc[data['PassengerId'] == passID, 'Family_Survival'] = 1
elif(smin==0.0):
data.loc[data['PassengerId'] == passID, 'Family_Survival'] = 0
print("Number of passenger with family/group survival information: "
+str(data[data['Family_Survival']!=0.5].shape[0]))
@tf.function
def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):
rotation = math.pi * rotation / 180.
 shear = math.pi * shear / 180.
 c1 = tf.math.cos(rotation)
 s1 = tf.math.sin(rotation)
 one = tf.constant([1],dtype='float32')
 zero = tf.constant([0],dtype='float32')
 rotation_matrix = tf.reshape(tf.concat([c1,s1,zero, -s1,c1,zero, zero,zero,one],axis=0),[3,3])
 c2 = tf.math.cos(shear)
 s2 = tf.math.sin(shear)
 shear_matrix = tf.reshape(tf.concat([one,s2,zero, zero,c2,zero, zero,zero,one],axis=0),[3,3])
 zoom_matrix = tf.reshape(tf.concat([one/height_zoom,zero,zero, zero,one/width_zoom,zero, zero,zero,one],axis=0),[3,3])
 shift_matrix = tf.reshape(tf.concat([one,zero,height_shift, zero,one,width_shift, zero,zero,one],axis=0),[3,3])
 return K.dot(K.dot(rotation_matrix, shear_matrix), K.dot(zoom_matrix, shift_matrix))
@tf.function
def transform(image):
DIM = IMAGE_SIZE[0]
XDIM = DIM%2
if tf.random.uniform(shape=[], minval=0, maxval=2, dtype=tf.int32, seed=SEED)== 0:
rot = 17.* tf.random.normal([1],dtype='float32')
 else:
rot = tf.constant([0],dtype='float32')
 if tf.random.uniform(shape=[], minval=0, maxval=2, dtype=tf.int32, seed=SEED)== 0:
shr = 5.5 * tf.random.normal([1],dtype='float32')
 else:
shr = tf.constant([0],dtype='float32')
 if tf.random.uniform(shape=[], minval=0, maxval=3, dtype=tf.int32, seed=SEED)== 0:
h_zoom = tf.random.normal([1],dtype='float32')/8.5
if h_zoom > 0:
h_zoom = 1.0 + h_zoom * -1
else:
h_zoom = 1.0 + h_zoom
else:
h_zoom = tf.constant([1],dtype='float32')
 if tf.random.uniform(shape=[], minval=0, maxval=3, dtype=tf.int32, seed=SEED)== 0:
w_zoom = tf.random.normal([1],dtype='float32')/8.5
if w_zoom > 0:
w_zoom = 1.0 + w_zoom * -1
else:
w_zoom = 1.0 + w_zoom
else:
w_zoom = tf.constant([1],dtype='float32')
 if tf.random.uniform(shape=[], minval=0, maxval=3, dtype=tf.int32, seed=SEED)== 0:
h_shift = 18.* tf.random.normal([1],dtype='float32')
 else:
h_shift = tf.constant([0],dtype='float32')
 if tf.random.uniform(shape=[], minval=0, maxval=3, dtype=tf.int32, seed=SEED)== 0:
w_shift = 18.* tf.random.normal([1],dtype='float32')
 else:
w_shift = tf.constant([0],dtype='float32')
 m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift)
 x = tf.repeat(tf.range(DIM//2,-DIM//2,-1), DIM)
 y = tf.tile(tf.range(-DIM//2,DIM//2),[DIM])
 z = tf.ones([DIM*DIM],dtype='int32')
 idx = tf.stack([x,y,z])
 idx2 = K.dot(m,tf.cast(idx,dtype='float32'))
idx2 = K.cast(idx2,dtype='int32')
 idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)
 idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])
 d = tf.gather_nd(image,tf.transpose(idx3))
return tf.reshape(d,[DIM,DIM,3] )<prepare_x_and_y>
data = data.reset_index(drop=True)
 data = data.drop('Survived', axis=1)
 data.tail()
@tf.function
def transform_grid_mark(image, inv_mat, image_shape):
h, w, c = image_shape
cx, cy = w//2, h//2
new_xs = tf.repeat(tf.range(-cx, cx, 1), h)
 new_ys = tf.tile(tf.range(-cy, cy, 1), [w])
 new_zs = tf.ones([h*w], dtype=tf.int32)
 old_coords = tf.matmul(inv_mat, tf.cast(tf.stack([new_xs, new_ys, new_zs]), tf.float32))
old_coords_x, old_coords_y = tf.round(old_coords[0, :] + tf.cast(w, tf.float32)//2.) , tf.round(old_coords[1, :] + tf.cast(h, tf.float32)//2.)
old_coords_x = tf.cast(old_coords_x, tf.int32)
 old_coords_y = tf.cast(old_coords_y, tf.int32)
 clip_mask_x = tf.logical_or(old_coords_x<0, old_coords_x>w-1)
 clip_mask_y = tf.logical_or(old_coords_y<0, old_coords_y>h-1)
 clip_mask = tf.logical_or(clip_mask_x, clip_mask_y)
 old_coords_x = tf.boolean_mask(old_coords_x, tf.logical_not(clip_mask))
old_coords_y = tf.boolean_mask(old_coords_y, tf.logical_not(clip_mask))
new_coords_x = tf.boolean_mask(new_xs+cx, tf.logical_not(clip_mask))
new_coords_y = tf.boolean_mask(new_ys+cy, tf.logical_not(clip_mask))
old_coords = tf.cast(tf.stack([old_coords_y, old_coords_x]), tf.int32)
 new_coords = tf.cast(tf.stack([new_coords_y, new_coords_x]), tf.int64)
 rotated_image_values = tf.gather_nd(image, tf.transpose(old_coords))
rotated_image_channel = list()
for i in range(c):
vals = rotated_image_values[:,i]
sparse_channel = tf.SparseTensor(tf.transpose(new_coords), vals, [h, w])
 rotated_image_channel.append(tf.sparse.to_dense(sparse_channel, default_value=0, validate_indices=False))
return tf.transpose(tf.stack(rotated_image_channel), [1,2,0])
 @tf.function
def random_rotate(image, angle, image_shape):
def get_rotation_mat_inv(angle):
angle = math.pi * angle / 180
cos_val = tf.math.cos(angle)
 sin_val = tf.math.sin(angle)
 one = tf.constant([1], tf.float32)
 zero = tf.constant([0], tf.float32)
 rot_mat_inv = tf.concat([cos_val, sin_val, zero,
-sin_val, cos_val, zero,
zero, zero, one], axis=0)
 rot_mat_inv = tf.reshape(rot_mat_inv, [3,3])
 return rot_mat_inv
angle = float(angle)* tf.random.normal([1],dtype='float32')
 rot_mat_inv = get_rotation_mat_inv(angle)
 return transform_grid_mark(image, rot_mat_inv, image_shape)
 @tf.function
def grid_mask() :
h = tf.constant(IMAGE_SIZE[0], dtype=tf.float32)
 w = tf.constant(IMAGE_SIZE[1], dtype=tf.float32)
 image_height, image_width =(h, w)
 d1 = 112
d2 = 352
rotate_angle = 45
ratio = 0.6
hh = tf.math.ceil(tf.math.sqrt(h*h+w*w))
hh = tf.cast(hh, tf.int32)
 hh = hh+1 if hh%2==1 else hh
d = tf.random.uniform(shape=[], minval=d1, maxval=d2, dtype=tf.int32)
 l = tf.cast(tf.cast(d,tf.float32)*ratio+0.5, tf.int32)
 st_h = tf.random.uniform(shape=[], minval=0, maxval=d, dtype=tf.int32)
 st_w = tf.random.uniform(shape=[], minval=0, maxval=d, dtype=tf.int32)
 y_ranges = tf.range(-1 * d + st_h, -1 * d + st_h + l)
 x_ranges = tf.range(-1 * d + st_w, -1 * d + st_w + l)
 for i in range(0, hh//d+1):
s1 = i * d + st_h
s2 = i * d + st_w
y_ranges = tf.concat([y_ranges, tf.range(s1,s1+l)], axis=0)
 x_ranges = tf.concat([x_ranges, tf.range(s2,s2+l)], axis=0)
 x_clip_mask = tf.logical_or(x_ranges <0 , x_ranges > hh-1)
 y_clip_mask = tf.logical_or(y_ranges <0 , y_ranges > hh-1)
 clip_mask = tf.logical_or(x_clip_mask, y_clip_mask)
 x_ranges = tf.boolean_mask(x_ranges, tf.logical_not(clip_mask))
y_ranges = tf.boolean_mask(y_ranges, tf.logical_not(clip_mask))
hh_ranges = tf.tile(tf.range(0,hh), [tf.cast(tf.reduce_sum(tf.ones_like(x_ranges)) , tf.int32)])
 x_ranges = tf.repeat(x_ranges, hh)
 y_ranges = tf.repeat(y_ranges, hh)
 y_hh_indices = tf.transpose(tf.stack([y_ranges, hh_ranges]))
x_hh_indices = tf.transpose(tf.stack([hh_ranges, x_ranges]))
y_mask_sparse = tf.SparseTensor(tf.cast(y_hh_indices, tf.int64), tf.zeros_like(y_ranges), [hh, hh])
 y_mask = tf.sparse.to_dense(y_mask_sparse, 1, False)
 x_mask_sparse = tf.SparseTensor(tf.cast(x_hh_indices, tf.int64), tf.zeros_like(x_ranges), [hh, hh])
 x_mask = tf.sparse.to_dense(x_mask_sparse, 1, False)
 mask = tf.expand_dims(tf.clip_by_value(x_mask + y_mask, 0, 1), axis=-1)
 mask = random_rotate(mask, rotate_angle, [hh, hh, 1])
 mask = tf.image.crop_to_bounding_box(mask,(hh-tf.cast(h, tf.int32)) //2,(hh-tf.cast(w, tf.int32)) //2, tf.cast(image_height, tf.int32), tf.cast(image_width, tf.int32))
return mask
@tf.function
def apply_grid_mask(image):
mask = grid_mask()
mask = tf.concat([mask, mask, mask], axis=-1)
 return image * tf.cast(mask, 'float32' )<normalization>
data['Fare'] = pd.qcut(data['Fare'], 4)
 lbl = LabelEncoder()
data['Fare'] = lbl.fit_transform(data['Fare'] )
@tf.function
def image_augmentation_tta_0(iw, filename):
max_chance = 8
image, words = iw
if tf.random.uniform(shape=[], minval=0, maxval=11, dtype=tf.int32, seed=SEED)< max_chance:
image = tf.image.flip_left_right(image)
 if tf.random.uniform(shape=[], minval=0, maxval=11, dtype=tf.int32, seed=SEED)< max_chance:
image = tf.image.random_brightness(image, 0.1)
 if tf.random.uniform(shape=[], minval=0, maxval=11, dtype=tf.int32, seed=SEED)< max_chance:
image = tf.image.random_contrast(image, 0.9, 1.1)
 return(( image, words), filename)
 @tf.function
def image_augmentation_tta_1(iw, filename):
max_chance = 8
image, words = iw
if tf.random.uniform(shape=[], minval=0, maxval=11, dtype=tf.int32, seed=SEED)< max_chance:
image = tf.image.flip_left_right(image)
 if tf.random.uniform(shape=[], minval=0, maxval=11, dtype=tf.int32, seed=SEED)< max_chance:
image = tf.image.random_brightness(image, 0.1)
 if tf.random.uniform(shape=[], minval=0, maxval=11, dtype=tf.int32, seed=SEED)< max_chance:
image = tf.image.random_contrast(image, 0.9, 1.1)
 if tf.random.uniform(shape=[], minval=0, maxval=11, dtype=tf.int32, seed=SEED)< max_chance:
image = tf.image.random_saturation(image, 0.95, 1.05)
 if tf.random.uniform(shape=[], minval=0, maxval=11, dtype=tf.int32, seed=SEED)< max_chance:
image = tf.image.random_hue(image, 0.05)
 if tf.random.uniform(shape=[], minval=0, maxval=11, dtype=tf.int32, seed=SEED)< max_chance:
image = transform(image)
 return(( image, words), filename)
 @tf.function
def image_augmentation_tta_2(iw, filename):
max_chance = 8
image, words = iw
if tf.random.uniform(shape=[], minval=0, maxval=11, dtype=tf.int32, seed=SEED)< max_chance:
image = tf.image.flip_left_right(image)
 if tf.random.uniform(shape=[], minval=0, maxval=11, dtype=tf.int32, seed=SEED)< max_chance:
image = tf.image.random_brightness(image, 0.1)
 if tf.random.uniform(shape=[], minval=0, maxval=11, dtype=tf.int32, seed=SEED)< max_chance:
image = tf.image.random_contrast(image, 0.9, 1.1)
 if tf.random.uniform(shape=[], minval=0, maxval=11, dtype=tf.int32, seed=SEED)< max_chance:
image = tf.image.random_saturation(image, 0.95, 1.05)
 if tf.random.uniform(shape=[], minval=0, maxval=11, dtype=tf.int32, seed=SEED)< max_chance:
image = tf.image.random_hue(image, 0.05)
 if tf.random.uniform(shape=[], minval=0, maxval=11, dtype=tf.int32, seed=SEED)< max_chance:
image = transform(image)
 if tf.random.uniform(shape=[], minval=0, maxval=10, dtype=tf.int32, seed=SEED)< max_chance:
image = apply_grid_mask(image)
 return(( image, words), filename )<import_modules>
titles_data = sorted(set([x for x in data['Name'].map(lambda x: get_title(x)) ]))
print(len(titles_data), ':', titles_data )
from tensorflow.keras.models import Model, Sequential
from tensorflow.keras.layers import Input, Flatten, Dense, Dropout, AveragePooling2D, GlobalAveragePooling2D, SpatialDropout2D, BatchNormalization, Activation, Concatenate<init_hyperparams>
data['Title'] = data['Name'].map(lambda x: get_title(x))
data['Title'] = data.apply(set_title, axis=1 )
class EpochCallback(tf.keras.callbacks.Callback):
def on_epoch_begin(self, epoch, logs=None):
global current_epoch
global chance
current_epoch = epoch
if current_epoch < 2:
chance = 0
elif current_epoch < 9:
chance = current_epoch - 1
else:
chance = 8
print(f'Epoch
print(datetime.now() )<choose_model_class>
print(data['Title'].value_counts() )
es_val_acc = tf.keras.callbacks.EarlyStopping(
 monitor='val_sparse_categorical_accuracy', min_delta=0.001, patience=5, verbose=1, mode='auto',
baseline=None, restore_best_weights=True 
)
 es_val_loss = tf.keras.callbacks.EarlyStopping(
 monitor='val_loss', min_delta=0.001, patience=5, verbose=1, mode='auto',
baseline=None, restore_best_weights=True 
)
 es_acc = tf.keras.callbacks.EarlyStopping(
 monitor='sparse_categorical_accuracy', min_delta=0.001, patience=5, verbose=1, mode='auto',
baseline=None, restore_best_weights=False 
)
 es_loss = tf.keras.callbacks.EarlyStopping(
 monitor='loss', min_delta=0.001, patience=5, verbose=1, mode='auto',
baseline=None, restore_best_weights=False 
)
 epoch_cb = EpochCallback()<choose_model_class>
print('Total missing age data: ', pd.isnull(data['Age'] ).sum())
with strategy.scope() :
efn7 = efn.EfficientNetB7(weights='noisy-student', include_top=False, input_shape=(IMAGE_SIZE[0], IMAGE_SIZE[1], 3))
for layer in efn7.layers:
layer.trainable = True
model_image = Sequential([
efn7,
GlobalAveragePooling2D(name='efficientnet-b7_gap'),
], name='b7-image')
 model_words = Sequential([
Input(( 6633,), name='mlp-words_input'),
Dense(331, name='mlp-words_dense_1'),
BatchNormalization(name='mlp-words_bn_1'),
Activation('relu', name='mlp-words_act_1'),
Dense(110, name='mlp-words_dense_2'),
BatchNormalization(name='mlp-words_bn_2'),
Activation('relu', name='mlp-words_act_2'),
], name='mlp-words')
 concatenate = Concatenate(name='concatenate' )([model_image.output, model_words.output])
 output = Dense(len(CLASSES), activation='softmax', name='output' )(concatenate)
 model = Model(inputs=[model_image.input, model_words.input], outputs=output)
 model.compile(optimizer=tfa.optimizers.LAMB(0.01), loss='sparse_categorical_crossentropy', metrics=['sparse_categorical_accuracy'])
 model.summary() 
<train_model>
data['Age'] = data.groupby('Title')['Age'].apply(lambda x: x.fillna(x.median()))
print(f'Pre training time : {(datetime.now() - PRE_TRAINING_TIME_START ).total_seconds() } seconds' )<init_hyperparams>
data['Age'] = pd.qcut(data['Age'], 4)
 lbl = LabelEncoder()
data['Age'] = lbl.fit_transform(data['Age'] )
LR_START = 0.0005
LR_MAX = 0.001
LR_MIN = 0.00015
LR_RAMPUP_EPOCHS = 2
LR_SUSTAIN_EPOCHS = 0
LR_EXP_DECAY = 0.83
def lrfn(epoch):
if epoch < LR_RAMPUP_EPOCHS:
lr =(LR_MAX - LR_START)/ LR_RAMPUP_EPOCHS * epoch + LR_START
elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:
lr = LR_MAX
else:
lr =(LR_MAX - LR_MIN)* LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS)+ LR_MIN
return lr
lr_schedule = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose = True)
 plt_lr(EPOCHS )<train_model>
data['Title'] = data['Title'].replace(['Mr', 'Miss', 'Mrs', 'Master'], [0, 1, 2, 3] )
model.fit(
 get_training_dataset(do_aug=DO_AUG), steps_per_epoch=STEPS_PER_EPOCH,
epochs=EPOCHS,
callbacks=[es_acc, epoch_cb, lr_schedule], verbose=1
)<compute_test_metric>
data['Sex'] = data['Sex'].replace(['male', 'female'], [0, 1] )
h = model.history
plt_acc(h)
 plt_loss(h )<feature_engineering>
data['Embarked'] = data['Embarked'].fillna(data['Embarked'].mode() [0])
 data['Embarked'] = data['Embarked'].replace(['S', 'C', 'Q'], [0, 1, 2] )
POST_TRAINING_TIME_START = datetime.now()<predict_on_test>
data['Cabin'] = data['Cabin'].map(lambda x: x[0] )
def test(tta=None):
test_ds = get_test_dataset(ordered=True, tta=tta)
 print(f'Computing predictions for TTA {tta}...')
 test_images_ds = test_ds.map(lambda iw, filename: [iw])
 model_pred = model.predict(test_images_ds)
 return model_pred<categorify>
data['Cabin'].value_counts()
model_pred = test(tta=None)
 model_pred_tta_0 = test(tta=0)
 model_pred_tta_1 = test(tta=1)
 model_pred_tta_2 = test(tta=2 )<prepare_output>
def unknown_cabin(cabin):
if cabin != 'U':
return 1
else:
return 0
data['Cabin'] = data['Cabin'].apply(lambda x:unknown_cabin(x))
pred_plain = np.argmax(model_pred, axis=-1)
 test_ds = get_test_dataset(ordered=True)
 test_ids_ds = test_ds.map(lambda iw, filename: filename ).unbatch()
test_ids = next(iter(test_ids_ds.batch(pred_plain.shape[0])) ).numpy().astype('U')
 df_submission = pd.DataFrame({'filename': test_ids, 'category': pred_plain})
 df_submission = df_submission.drop_duplicates()
df_submission['category'] = df_submission['category'].apply(lambda c: str(c ).zfill(2))
df_submission<save_to_csv>
data['FamilySize'] = data['SibSp'] + data['Parch'] + 1
data['IsAlone'] = 1
data['IsAlone'].loc[data['FamilySize'] > 1] = 0
df_submission.to_csv('submission.csv', index=False)
 !head submission.csv<feature_engineering>
data = data.drop(['Name', 'Parch', 'SibSp', 'Ticket', 'Last_Name', 'PassengerId'], axis = 1 )
pred_tta_0 = np.mean(np.array([model_pred, model_pred_tta_0]), axis=0)
 pred_tta_0 = np.argmax(pred_tta_0, axis=-1)
 test_ds = get_test_dataset(ordered=True)
 test_ids_ds = test_ds.map(lambda iw, filename: filename ).unbatch()
test_ids = next(iter(test_ids_ds.batch(pred_tta_0.shape[0])) ).numpy().astype('U')
 df_submission_tta_0 = pd.DataFrame({'filename': test_ids, 'category': pred_tta_0})
 df_submission_tta_0 = df_submission_tta_0.drop_duplicates()
df_submission_tta_0['category'] = df_submission_tta_0['category'].apply(lambda c: str(c ).zfill(2))
df_submission_tta_0<save_to_csv>
train = data[:ntrain]
test = data[ntrain:]
df_submission_tta_0.to_csv('submission_tta_0.csv', index=False)
 !head submission_tta_0.csv<feature_engineering>
X_test = test
X_train = train
scaler = StandardScaler()
X_train = scaler.fit_transform(X_train)
 X_test = scaler.transform(X_test )
pred_tta_all = np.mean(np.array([model_pred, model_pred_tta_0, model_pred_tta_1, model_pred_tta_2]), axis=0)
 pred_tta_all = np.argmax(pred_tta_all, axis=-1)
 test_ds = get_test_dataset(ordered=True)
 test_ids_ds = test_ds.map(lambda iw, filename: filename ).unbatch()
test_ids = next(iter(test_ids_ds.batch(pred_tta_all.shape[0])) ).numpy().astype('U')
 df_submission_tta_all = pd.DataFrame({'filename': test_ids, 'category': pred_tta_all})
 df_submission_tta_all = df_submission.drop_duplicates()
df_submission_tta_all['category'] = df_submission_tta_all['category'].apply(lambda c: str(c ).zfill(2))
df_submission_tta_all<save_to_csv>
ran = RandomForestClassifier(random_state=1)
 knn = KNeighborsClassifier()
log = LogisticRegression()
xgb = XGBClassifier()
gbc = GradientBoostingClassifier()
svc = SVC(probability=True)
 ext = ExtraTreesClassifier()
ada = AdaBoostClassifier()
gnb = GaussianNB()
gpc = GaussianProcessClassifier()
bag = BaggingClassifier()
models = [ran, knn, log, xgb, gbc, svc, ext, ada, gnb, gpc, bag]
scores = []
for mod in models:
mod.fit(X_train, y_train)
 acc = cross_val_score(mod, X_train, y_train, scoring = "accuracy", cv = 10)
 scores.append(acc.mean() )
df_submission_tta_all.to_csv('submission_tta_all.csv', index=False)
 !head submission_tta_all.csv<load_pretrained>
results = pd.DataFrame({
'Model': ['Random Forest', 'K Nearest Neighbour', 'Logistic Regression', 'XGBoost', 'Gradient Boosting', 'SVC', 'Extra Trees', 'AdaBoost', 'Gaussian Naive Bayes', 'Gaussian Process', 'Bagging Classifier'],
'Score': scores})
 result_df = results.sort_values(by='Score', ascending=False ).reset_index(drop=True)
 result_df.head(11 )
model.save('model.h5')
 model.save_weights('model_weights.h5' )<train_model>
fi = {'Features':train.columns.tolist() , 'Importance':xgb.feature_importances_}
importance = pd.DataFrame(fi, index=None ).sort_values('Importance', ascending=False )
print(f'Post training time : {(datetime.now() - POST_TRAINING_TIME_START ).total_seconds() } seconds' )<set_options>
fi = {'Features':train.columns.tolist() , 'Importance':np.transpose(log.coef_[0])}
importance = pd.DataFrame(fi, index=None ).sort_values('Importance', ascending=False )
%matplotlib inline
warnings.filterwarnings('ignore')
 warnings.filterwarnings('ignore', category=DeprecationWarning)
 pd.options.display.max_columns = 100
sns.set(rc={'figure.figsize':(12,9)})
<import_modules>
gbc_imp = pd.DataFrame({'Feature':train.columns, 'gbc importance':gbc.feature_importances_})
 xgb_imp = pd.DataFrame({'Feature':train.columns, 'xgb importance':xgb.feature_importances_})
 ran_imp = pd.DataFrame({'Feature':train.columns, 'ran importance':ran.feature_importances_})
 ext_imp = pd.DataFrame({'Feature':train.columns, 'ext importance':ext.feature_importances_})
 ada_imp = pd.DataFrame({'Feature':train.columns, 'ada importance':ada.feature_importances_})
 importances = gbc_imp.merge(xgb_imp, on='Feature' ).merge(ran_imp, on='Feature' ).merge(ext_imp, on='Feature' ).merge(ada_imp, on='Feature')
 importances['Average'] = importances.mean(axis=1)
 importances = importances.sort_values(by='Average', ascending=False ).reset_index(drop=True)
 importances
from xgboost import XGBClassifier
import xgboost as xgb<import_modules>
fi = {'Features':importances['Feature'], 'Importance':importances['Average']}
importance = pd.DataFrame(fi, index=None ).sort_values('Importance', ascending=False )
from sklearn.preprocessing import StandardScaler<load_from_csv>
train = train.drop(['Embarked','IsAlone'], axis=1)
 test = test.drop(['Embarked', 'IsAlone'], axis=1)
 X_train = train
X_test = test
X_train = scaler.fit_transform(X_train)
 X_test = scaler.transform(X_test )
data = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
<string_transform>
ran = RandomForestClassifier(random_state=1)
 knn = KNeighborsClassifier()
log = LogisticRegression()
xgb = XGBClassifier(random_state=1)
 gbc = GradientBoostingClassifier(random_state=1)
 svc = SVC(probability=True)
 ext = ExtraTreesClassifier(random_state=1)
 ada = AdaBoostClassifier(random_state=1)
 gnb = GaussianNB()
gpc = GaussianProcessClassifier()
bag = BaggingClassifier(random_state=1)
 models = [ran, knn, log, xgb, gbc, svc, ext, ada, gnb, gpc, bag]
scores_v2 = []
for mod in models:
mod.fit(X_train, y_train)
 acc = cross_val_score(mod, X_train, y_train, scoring = "accuracy", cv = 10)
 scores_v2.append(acc.mean() )
df=data
datetime=df.Dates.str.split(pat=" ",expand=True)
 datetime.columns=['Date','Time']
df_test=test
datetime_test=df_test.Dates.str.split(pat=" ",expand=True)
 datetime_test.columns=['Date','Time']<feature_engineering>
results = pd.DataFrame({
'Model': ['Random Forest', 'K Nearest Neighbour', 'Logistic Regression', 'XGBoost', 'Gradient Boosting', 'SVC', 'Extra Trees', 'AdaBoost', 'Gaussian Naive Bayes', 'Gaussian Process', 'Bagging Classifier'],
'Original Score': scores,
'Score with feature selection': scores_v2})
 result_df = results.sort_values(by='Score with feature selection', ascending=False ).reset_index(drop=True)
 result_df.head(11 )
Date=datetime.Date.str.split(pat="-",expand=True)
 Date.columns=['Year','Month','Day']
Time=datetime.Time.str.split(pat=":",expand=True)
 Time.columns=['Hour','Minute','Second']
Date_test=datetime_test.Date.str.split(pat="-",expand=True)
 Date_test.columns=['Year','Month','Day']
Time_test=datetime_test.Time.str.split(pat=":",expand=True)
 Time_test.columns=['Hour','Minute','Second']<concatenate>
n_estimators = [10, 25, 50, 75, 100]
max_depth = [3, None]
max_features = [1, 3, 5, 7]
min_samples_split = [2, 4, 6, 8, 10]
min_samples_leaf = [2, 4, 6, 8, 10]
hyperparams = {'n_estimators': n_estimators, 'max_depth': max_depth, 'max_features': max_features,
'min_samples_split': min_samples_split, 'min_samples_leaf': min_samples_leaf}
gd=GridSearchCV(estimator = RandomForestClassifier() , param_grid = hyperparams,
verbose=True, cv=5, scoring = "accuracy")
 gd.fit(X_train, y_train)
 print(gd.best_score_)
 print(gd.best_estimator_ )
df=pd.concat([df,Date,Time],axis=1)
 df_test=pd.concat([df_test,Date_test,Time_test],axis=1)
<drop_column>
n_neighbors = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 14, 16, 18, 20]
algorithm = ['auto']
weights = ['uniform', 'distance']
leaf_size = [1, 2, 3, 4, 5, 10, 15, 20, 25, 30]
hyperparams = {'algorithm': algorithm, 'weights': weights, 'leaf_size': leaf_size,
'n_neighbors': n_neighbors}
gd=GridSearchCV(estimator = KNeighborsClassifier() , param_grid = hyperparams,
verbose=True, cv=5, scoring = "accuracy")
 gd.fit(X_train, y_train)
 print(gd.best_score_)
 print(gd.best_estimator_ )
df=df.drop(labels=['Dates'],axis=1)
 df_test=df_test.drop(labels=['Dates'],axis=1)
<categorify>
penalty = ['l1', 'l2']
C = np.logspace(0, 4, 10)
 hyperparams = {'penalty': penalty, 'C': C}
gd=GridSearchCV(estimator = LogisticRegression() , param_grid = hyperparams,
verbose=True, cv=5, scoring = "accuracy")
 gd.fit(X_train, y_train)
 print(gd.best_score_)
 print(gd.best_estimator_ )
le = preprocessing.LabelEncoder()<prepare_x_and_y>
learning_rate = [0.0001, 0.0005, 0.001, 0.005, 0.01, 0.05, 0.1, 0.2]
n_estimators = [10, 25, 50, 75, 100, 250, 500, 750, 1000]
hyperparams = {'learning_rate': learning_rate, 'n_estimators': n_estimators}
gd=GridSearchCV(estimator = XGBClassifier() , param_grid = hyperparams,
verbose=True, cv=5, scoring = "accuracy")
 gd.fit(X_train, y_train)
 print(gd.best_score_)
 print(gd.best_estimator_ )
le_res=le.fit_transform(df['Category'])
 y=pd.DataFrame(le_res)
 y.columns=['Category'] 
<feature_engineering>
max_depth = [3, 4, 5, 6, 7, 8, 9, 10]
min_child_weight = [1, 2, 3, 4, 5, 6]
hyperparams = {'max_depth': max_depth, 'min_child_weight': min_child_weight}
gd=GridSearchCV(estimator = XGBClassifier(learning_rate=0.0001, n_estimators=10), param_grid = hyperparams,
verbose=True, cv=5, scoring = "accuracy")
 gd.fit(X_train, y_train)
 print(gd.best_score_)
 print(gd.best_estimator_ )
df["rot60_X"]=(0.5)* df["Y"] +(1.732/2)* df["X"]
df["rot60_Y"]=0.5 * df["Y"] -(1.732/2)* df["X"]
df_test["rot60_X"]=(0.5)* df_test["Y"] +(1.732/2)* df_test["X"]
df_test["rot60_Y"]=0.5 * df_test["Y"] -(1.732/2)* df_test["X"]
df["rot45_X"]=0.707 * df["Y"] + 0.707 * df["X"]
df["rot45_Y"]=0.707 * df["Y"] - 0.707 * df["X"]
df_test["rot45_X"]=0.707 * df_test["Y"] + 0.707 * df_test["X"]
df_test["rot45_Y"]=0.707 * df_test["Y"] - 0.707 * df_test["X"]
df["rot30_X"]=(1.732/2)* df["Y"] + 0.5 * df["X"]
df["rot30_Y"]=(1.732/2)* df["Y"] - 0.5 * df["X"]
df_test["rot30_X"]=(1.732/2)* df_test["Y"] + 0.5 * df_test["X"]
df_test["rot30_Y"]=(1.732/2)* df_test["Y"] - 0.5 * df_test["X"] 
<feature_engineering>
gamma = [i*0.1 for i in range(0,5)]
hyperparams = {'gamma': gamma}
gd=GridSearchCV(estimator = XGBClassifier(learning_rate=0.0001, n_estimators=10, max_depth=3,
min_child_weight=1), param_grid = hyperparams,
verbose=True, cv=5, scoring = "accuracy")
 gd.fit(X_train, y_train)
 print(gd.best_score_)
 print(gd.best_estimator_ )
df["radial60"]=np.sqrt(np.power(df['rot60_X'],2)+ np.power(df['rot60_Y'],2))
df_test["radial60"]=np.sqrt(np.power(df_test['rot60_X'],2)+ np.power(df_test['rot60_Y'],2))<drop_column>
subsample = [0.6, 0.65, 0.7, 0.75, 0.8, 0.85, 0.9, 0.95, 1]
colsample_bytree = [0.6, 0.65, 0.7, 0.75, 0.8, 0.85, 0.9, 0.95, 1]
hyperparams = {'subsample': subsample, 'colsample_bytree': colsample_bytree}
gd=GridSearchCV(estimator = XGBClassifier(learning_rate=0.0001, n_estimators=10, max_depth=3,
min_child_weight=1, gamma=0), param_grid = hyperparams,
verbose=True, cv=5, scoring = "accuracy")
 gd.fit(X_train, y_train)
 print(gd.best_score_)
 print(gd.best_estimator_ )
df=df.drop(labels='rot60_X',axis=1)
 df_test=df_test.drop(labels='rot60_X',axis=1 )<drop_column>
reg_alpha = [1e-5, 1e-2, 0.1, 1, 100]
hyperparams = {'reg_alpha': reg_alpha}
gd=GridSearchCV(estimator = XGBClassifier(learning_rate=0.0001, n_estimators=10, max_depth=3,
min_child_weight=1, gamma=0, subsample=0.6, colsample_bytree=0.9),
param_grid = hyperparams, verbose=True, cv=5, scoring = "accuracy")
 gd.fit(X_train, y_train)
 print(gd.best_score_)
 print(gd.best_estimator_ )
df=df.drop(labels='rot60_Y',axis=1)
 df_test=df_test.drop(labels='rot60_Y',axis=1 )<drop_column>
learning_rate = [0.0001, 0.0005, 0.001, 0.005, 0.01, 0.05, 0.1, 0.2]
n_estimators = [100, 250, 500, 750, 1000, 1250, 1500]
hyperparams = {'learning_rate': learning_rate, 'n_estimators': n_estimators}
gd=GridSearchCV(estimator = GradientBoostingClassifier() , param_grid = hyperparams,
verbose=True, cv=5, scoring = "accuracy")
 gd.fit(X_train, y_train)
 print(gd.best_score_)
 print(gd.best_estimator_ )
df=df.drop(labels='Second',axis=1)
 df_test=df_test.drop(labels='Second',axis=1 )<feature_engineering>
Cs = [0.001, 0.01, 0.1, 1, 5, 10, 15, 20, 50, 100]
gammas = [0.001, 0.01, 0.1, 1]
hyperparams = {'C': Cs, 'gamma' : gammas}
gd=GridSearchCV(estimator = SVC(probability=True), param_grid = hyperparams,
verbose=True, cv=5, scoring = "accuracy")
 gd.fit(X_train, y_train)
 print(gd.best_score_)
 print(gd.best_estimator_ )
df['Minute']=df['Minute'].apply(lambda x:int(x))
df['Minute']=df['Minute'].apply(lambda x : 'low' if x <31 else 'high')
 df_test['Minute']=df_test['Minute'].apply(lambda x:int(x))
df_test['Minute']=df_test['Minute'].apply(lambda x : 'low' if x <31 else 'high')
<feature_engineering>
n_estimators = [10, 25, 50, 75, 100]
max_depth = [3, None]
max_features = [1, 3, 5, 7]
min_samples_split = [2, 4, 6, 8, 10]
min_samples_leaf = [2, 4, 6, 8, 10]
hyperparams = {'n_estimators': n_estimators, 'max_depth': max_depth, 'max_features': max_features,
'min_samples_split': min_samples_split, 'min_samples_leaf': min_samples_leaf}
gd=GridSearchCV(estimator = ExtraTreesClassifier() , param_grid = hyperparams,
verbose=True, cv=5, scoring = "accuracy")
 gd.fit(X_train, y_train)
 print(gd.best_score_)
 print(gd.best_estimator_ )
df['DayOfWeek']= df['DayOfWeek'].apply(lambda x : 'WeekHigh' if x in('Wednesday','Friday')else('WeekMed' if x in('Tuesday','Thursday','Saturday')else 'WeekLow'))
df_test['DayOfWeek']= df_test['DayOfWeek'].apply(lambda x : 'WeekHigh' if x in('Wednesday','Friday')else('WeekMed' if x in('Tuesday','Thursday','Saturday')else 'WeekLow')) 
<feature_engineering>
n_estimators = [10, 25, 50, 75, 100, 125, 150, 200]
learning_rate = [0.001, 0.01, 0.1, 0.5, 1, 1.5, 2]
hyperparams = {'n_estimators': n_estimators, 'learning_rate': learning_rate}
gd=GridSearchCV(estimator = AdaBoostClassifier() , param_grid = hyperparams,
verbose=True, cv=5, scoring = "accuracy")
 gd.fit(X_train, y_train)
 print(gd.best_score_)
 print(gd.best_estimator_ )
df['Intersection']=df['Address'].apply(lambda x : 1 if '/' in x else 0)
 df['Block']=df['Address'].apply(lambda x : 1 if 'Block' in x else 0)
 df_test['Intersection']=df_test['Address'].apply(lambda x : 1 if '/' in x else 0)
 df_test['Block']=df_test['Address'].apply(lambda x : 1 if 'Block' in x else 0 )<feature_engineering>
n_restarts_optimizer = [0, 1, 2, 3]
max_iter_predict = [1, 2, 5, 10, 20, 35, 50, 100]
warm_start = [True, False]
hyperparams = {'n_restarts_optimizer': n_restarts_optimizer, 'max_iter_predict': max_iter_predict, 'warm_start': warm_start}
gd=GridSearchCV(estimator = GaussianProcessClassifier() , param_grid = hyperparams,
verbose=True, cv=5, scoring = "accuracy")
 gd.fit(X_train, y_train)
 print(gd.best_score_)
 print(gd.best_estimator_ )
address=pd.DataFrame(df['Address'],columns=['Address'])
 address=address.Address.str.split(pat=" /",expand=True)
 address.columns=['Address','Intr2']
address=address.Address.str.split(pat=" /",expand=True)
 address.columns=['Address']
string=address.iloc[:,0]
string=string.str.strip()
address_fram=string.to_frame()
temp=address_fram['Address'].astype(str ).str[-2:]
address=temp.to_frame()
address['Address']=address['Address'].apply(lambda x :(x if x in("ST","AV","LN","DR","BL","HY","CT","RD","PL","PZ","TR","AL","CR","WK","EX","RW")else(( "I-80" if x in("80")else("HWY" if x in("WY")else("WAY" if x in("AY")else("TER" if x in("ER")else("ALMS" if x in("MS")else("MAR" if x in("AR")else("PARK" if x in("RK")else("STWY" if x in("WY")else("VIA" if x in("NO")else("BLOCK")))))))))))))
 df=df.drop(labels=['Address'],axis=1)
 df=pd.concat([address,df],axis=1)
<feature_engineering>
n_estimators = [10, 15, 20, 25, 50, 75, 100, 150]
max_samples = [1, 2, 3, 5, 7, 10, 15, 20, 25, 30, 50]
max_features = [1, 3, 5, 7]
hyperparams = {'n_estimators': n_estimators, 'max_samples': max_samples, 'max_features': max_features}
gd=GridSearchCV(estimator = BaggingClassifier() , param_grid = hyperparams,
verbose=True, cv=5, scoring = "accuracy")
 gd.fit(X_train, y_train)
 print(gd.best_score_)
 print(gd.best_estimator_ )
address_test=pd.DataFrame(df_test['Address'],columns=['Address'])
 address_test=address_test.Address.str.split(pat=" /",expand=True)
 address_test.columns=['Address','Intr2']
address_test=address_test.Address.str.split(pat=" /",expand=True)
 address_test.columns=['Address']
string_test=address_test.iloc[:,0]
string_test=string_test.str.strip()
address_fram_test=string_test.to_frame()
temp_test=address_fram_test['Address'].astype(str ).str[-2:]
address_test=temp_test.to_frame()
address_test['Address']=address_test['Address'].apply(lambda x :(x if x in("ST","AV","LN","DR","BL","HY","CT","RD","PL","PZ","TR","AL","CR","WK","EX","RW")else(( "I-80" if x in("80")else("HWY" if x in("WY")else("WAY" if x in("AY")else("TER" if x in("ER")else("ALMS" if x in("MS")else("MAR" if x in("AR")else("PARK" if x in("RK")else("STWY" if x in("WY")else("VIA" if x in("NO")else("BLOCK")))))))))))))
 df_test=df_test.drop(labels=['Address'],axis=1)
 df_test=pd.concat([address_test,df_test],axis=1 )<drop_column>
ran = RandomForestClassifier(n_estimators=50,
max_depth=3,
max_features=7,
min_samples_leaf=8,
min_samples_split=6,
random_state=1)
 knn = KNeighborsClassifier(algorithm='auto',
leaf_size=3,
n_neighbors=10,
weights='uniform')
 log = LogisticRegression(C=21.544346900318832,
penalty='l2')
 xgb = XGBClassifier(learning_rate=0.0001,
n_estimators=10,
random_state=1)
 gbc = GradientBoostingClassifier(learning_rate=0.0005,
n_estimators=1250,
random_state=1)
 svc = SVC(C=50, gamma=0.01, probability=True)
 ext = ExtraTreesClassifier(max_depth=3,
max_features=7,
min_samples_leaf=8,
min_samples_split=4,
n_estimators=25,
random_state=1)
 ada = AdaBoostClassifier(learning_rate=0.5,
n_estimators=25,
random_state=1)
 gpc = GaussianProcessClassifier(max_iter_predict=1)
 bag = BaggingClassifier(max_features=7, max_samples=50, n_estimators=20,random_state=1)
 models = [ran, knn, log, xgb, gbc, svc, ext, ada, gnb, gpc, bag]
scores_v3 = []
for mod in models:
mod.fit(X_train, y_train)
 acc = cross_val_score(mod, X_train, y_train, scoring = "accuracy", cv = 10)
 scores_v3.append(acc.mean() )
Id=df['Id']
df=df.drop(['Descript','Resolution','Id'],axis=1)
 Id_test=df_test['Id']
df_test=df_test.drop(['Descript','Resolution','Id'],axis=1 )<categorify>
results = pd.DataFrame({
'Model': ['Random Forest', 'K Nearest Neighbour', 'Logistic Regression', 'XGBoost', 'Gradient Boosting', 'SVC', 'Extra Trees', 'AdaBoost', 'Gaussian Naive Bayes', 'Gaussian Process', 'Bagging Classifier'],
'Original Score': scores,
'Score with feature selection': scores_v2,
'Score with tuned parameters': scores_v3})
 result_df = results.sort_values(by='Score with tuned parameters', ascending=False ).reset_index(drop=True)
 result_df.head(11 )
le = preprocessing.LabelEncoder()<categorify>
grid_hard = VotingClassifier(estimators = [('Random Forest', ran), 
('Logistic Regression', log), 
('XGBoost', xgb), 
('Gradient Boosting', gbc), 
('Extra Trees', ext), 
('AdaBoost', ada), 
('Gaussian Process', gpc), 
('SVC', svc), 
('K Nearest Neighbour', knn), 
('Bagging Classifier', bag)], voting = 'hard')
 grid_hard_cv = model_selection.cross_validate(grid_hard, X_train, y_train, cv = 10, return_train_score=True)
 grid_hard.fit(X_train, y_train)
 print("Hard voting on train set score mean: {:.2f}".format(grid_hard_cv['train_score'].mean() *100))
print("Hard voting on test set score mean: {:.2f}".format(grid_hard_cv['test_score'].mean() *100))
le_res=le.fit_transform(df['DayOfWeek'])
 Day=pd.DataFrame(le_res)
 Day.columns=['DayOfWeek']
df=df.drop(labels=['DayOfWeek'],axis=1)
 df=pd.concat([Day,df],axis=1)
 le_res_test=le.fit_transform(df_test['DayOfWeek'])
 Day_test=pd.DataFrame(le_res_test)
 Day_test.columns=['DayOfWeek']
df_test=df_test.drop(labels=['DayOfWeek'],axis=1)
 df_test=pd.concat([Day_test,df_test],axis=1 )<categorify>
grid_soft = VotingClassifier(estimators = [('Random Forest', ran), 
('Logistic Regression', log), 
('XGBoost', xgb), 
('Gradient Boosting', gbc), 
('Extra Trees', ext), 
('AdaBoost', ada), 
('Gaussian Process', gpc), 
('SVC', svc), 
('K Nearest Neighbour', knn), 
('Bagging Classifier', bag)], voting = 'soft')
 grid_soft_cv = model_selection.cross_validate(grid_soft, X_train, y_train, cv = 10, return_train_score=True)
 grid_soft.fit(X_train, y_train)
 print("Soft voting on train set score mean: {:.2f}".format(grid_soft_cv['train_score'].mean() *100))
print("Soft voting on test set score mean: {:.2f}".format(grid_soft_cv['test_score'].mean() *100))
le_res=le.fit_transform(df['PdDistrict'])
 District=pd.DataFrame(le_res)
 District.columns=['District']
df=df.drop(labels=['PdDistrict'],axis=1)
 df=pd.concat([District,df],axis=1)
 le_res_test=le.fit_transform(df_test['PdDistrict'])
 District_test=pd.DataFrame(le_res_test)
 District_test.columns=['District']
df_test=df_test.drop(labels=['PdDistrict'],axis=1)
 df_test=pd.concat([District_test,df_test],axis=1)
<categorify>
predictions = grid_hard.predict(X_test)
 submission = pd.concat([pd.DataFrame(passId), pd.DataFrame(predictions)], axis = 'columns')
 submission.columns = ["PassengerId", "Survived"]
submission.to_csv('titanic_submission.csv', header = True, index = False )