Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<drop_column>
pd.options.mode.chained_assignment = None
warnings.filterwarnings('ignore' )
train_array = train.drop(columns=['matchId', 'groupId', 'winPlacePerc'] ).values<define_variables>
train = pd.read_csv('.. /input/train.csv', header=0)
 test = pd.read_csv('.. /input/test.csv', header=0)
 test.insert(1,'Survived',np.nan)
 all = pd.concat([train, test] )
group_col_inds = [train.columns.drop(['matchId', 'groupId', 'winPlacePerc'] ).get_loc(col)for col in group_cols]
other_col_inds = [train.columns.drop(['matchId', 'groupId', 'winPlacePerc'] ).get_loc(col)for col in other_cols]
def get_comp_batch(inds, batch_size, omit_last=True):
rand_inds = np.random.permutation(inds.shape[0])
 if omit_last:
func = np.floor
else:
func = np.ceil
for batch_n in range(int(func(inds.shape[0]/batch_size))):
batch_inds = inds[rand_inds[batch_n*batch_size:(batch_n+1)*batch_size]]
X1_batch = train_array[batch_inds[:,0]][:,group_col_inds]
X2_batch = train_array[batch_inds[:,1]][:,group_col_inds]
X3_batch = train_array[batch_inds[:,0]][:,other_col_inds]
yield X1_batch, X2_batch, X3_batch<categorify>
all['Title'] = all.Name.str.extract('([A-Za-z]+)\.', expand=False )
dropout_rate = [0.0, 0.0, 0.0, 0.0, 0.1, 0.2, 0.2, 0.2]
layer_sizes = [100, 100, 60, 60, 60, 30, 30, 30]
tf.reset_default_graph()
def leaky_relu(z, name=None):
return tf.maximum(0.01*z, z, name=name)
 X1 = tf.placeholder(dtype=tf.float32, shape=[None, len(group_cols)], name='X1')
 X2 = tf.placeholder(dtype=tf.float32, shape=[None, len(group_cols)], name='X2')
 X3 = tf.placeholder(dtype=tf.float32, shape=[None, len(other_cols)], name='X3')
 X = tf.concat([tf.concat([X1, X2, X3], axis=1), tf.concat([X2, X1, X3], axis=1)], axis=0)
 y = tf.placeholder(dtype=tf.float32, shape=(None), name='y')
 y_all = tf.concat([y, 1-y], axis=0)
 training = tf.placeholder_with_default(False, shape=(None), name='training')
 with tf.name_scope('layers')as scope:
hidden = tf.layers.dropout(X, rate=dropout_rate[0], training=training, name='input_dropout')
 for i in range(8):
hidden = tf.layers.dense(hidden, layer_sizes[i], name=f'hidden_{i}',
kernel_initializer=tf.contrib.layers.variance_scaling_initializer())
 hidden = tf.layers.batch_normalization(hidden, training=training, momentum=0.9, name=f'hidden_{i}_bn')
 hidden = tf.nn.elu(hidden, name=f'hidden_{i}_activation')
 hidden = leaky_relu(hidden, name=f'hidden_{i}_activation')
 hidden = tf.layers.dropout(hidden, rate=dropout_rate[i], training=training, name=f'dropout_{i}')
 logits = tf.layers.dense(hidden, 1, name='logits',
kernel_initializer=tf.contrib.layers.variance_scaling_initializer())
 output = tf.nn.sigmoid(logits, name='output' )<choose_model_class>
all.loc[all['Title'].isin(['Ms','Mlle']), 'Title'] = 'Miss'
all.loc[all['Title'].isin(['Mme','Lady','Dona','Countess']), 'Title'] = 'Mrs'
all.loc[all['Title'].isin(['Col','Major','Sir','Rev','Capt','Don','Jonkheer']), 'Title'] = 'Mr'
all.loc[(all['Title'] == 'Dr')&(all['Sex'] == 'male'),'Title'] = 'Mr'
all.loc[(all['Title'] == 'Dr')&(all['Sex'] == 'female'),'Title'] = 'Mrs'
with tf.name_scope('loss')as scope:
xentropy = tf.nn.sigmoid_cross_entropy_with_logits(labels=y_all,logits=tf.reshape(logits, [-1]),name='xentropy')
 loss = tf.reduce_mean(xentropy)
 with tf.name_scope('training')as scope:
lr_low = 0.00001
lr_high = 0.003
lr_high_2 = 0.0003
decay_rate = 15
lr = tf.Variable(lr_high, trainable=False, name='learning_rate')
 optimizer = tf.train.AdamOptimizer(learning_rate=lr, epsilon=1e-10)
 training_op = optimizer.minimize(loss)
 with tf.name_scope('evaluation')as scope:
accuracy = tf.reduce_mean(tf.cast(tf.equal(tf.round(tf.reshape(output,[-1])) ,y_all), tf.float32), name='accuracy')
 def get_full_set_accuracy(sess, inds, batch_size=1000):
y_batch = np.ones(shape=[batch_size])
 y_for_comp = np.r_[y_batch, 1 - y_batch]
acc_data = np.empty(shape=0)
 for X1_batch, X2_batch, X3_batch in get_comp_batch(inds=inds, batch_size=batch_size):
feed_dict = {X1:X1_batch, X2:X2_batch, X3:X3_batch, y:y_batch}
y_pred = sess.run(output, feed_dict=feed_dict ).reshape(-1)
 batch_acc_data =(( y_pred > 0.5 ).astype(np.float32)== y_for_comp ).astype(np.float32)
 acc_data = np.r_[acc_data, batch_acc_data]
return np.mean(acc_data)
 extra_training_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
 init = tf.global_variables_initializer()
saver_comp = tf.train.Saver()<define_variables>
all['FamSize'] = all.apply(lambda s: 1+s['SibSp']+s['Parch'], axis = 1)
 all['isAlone'] = all.apply(lambda s: 1 if s['FamSize'] == 1 else 0, axis = 1 )
gc.collect()
batch_size = 500
n_epochs = 1000
max_epochs_without_improving = 50
max_epochs_wo_lr_change = 7
max_time = 5.5
acc_data = {'acc_train':[], 'acc_val':[]}
batch_acc_data = []
batch_lr_data = []
start_time = datetime.datetime.now()
with tf.Session() as sess:
init.run()
max_val_acc = 0
epochs_wo_improvement = 0
epochs_since_lr_update = 0
gap = 1
batch_lr = lr_high
y_batch = np.ones(shape=[batch_size])
 for epoch in range(n_epochs):
n_batch = 0
if epoch == 15:
batch_size = 1000
y_batch = np.ones(shape=[batch_size])
 max_epochs_without_improving = 100
max_epochs_wo_lr_change = 10
if epoch == 25:
batch_size = 2000
y_batch = np.ones(shape=[batch_size])
 if epoch == 40:
batch_size = 4000
y_batch = np.ones(shape=[batch_size])
 max_epochs_wo_lr_change = 15
if epoch == 60:
batch_size = 8000
y_batch = np.ones(shape=[batch_size])
 max_epochs_wo_lr_change = 20
for X1_batch, X2_batch, X3_batch in get_comp_batch(inds=train_comp_inds, batch_size=batch_size):
feed_dict = {X1:X1_batch, X2:X2_batch, X3:X3_batch, y:y_batch}
batch_lr_data.append(batch_lr)
 batch_acc_data.append(sess.run(accuracy, feed_dict=feed_dict))
feed_dict = {X1:X1_batch, X2:X2_batch, X3:X3_batch, y:y_batch, training:True, lr:batch_lr}
sess.run([training_op, extra_training_ops], feed_dict=feed_dict)
 n_batch += 1
if(epoch >= 20)and(epoch % 1 == 0):
curr_ids = np.random.permutation(train_comp_inds.shape[0])[:100000]
train_acc = get_full_set_accuracy(sess, train_comp_inds[curr_ids], batch_size=10000)
 acc_data['acc_train'].append(train_acc)
 print('epoch', epoch, 'time passed:',(datetime.datetime.now() - start_time), 'max lr:', batch_lr)
 print('Accuracy: train:', train_acc)
 if train_acc > max_val_acc:
epochs_wo_improvement = 0
epochs_since_lr_update = 0
max_val_acc = train_acc
save_path_comp = saver_comp.save(sess, '.. /DNN_data/dnn_state_comp.ckpt')
 print('- best so far!')
 else:
epochs_wo_improvement += 1
epochs_since_lr_update += 1
if(epochs_wo_improvement > max_epochs_without_improving)or(( datetime.datetime.now() - kernel_start_time ).seconds/60/60 > max_time):
print('early breaking!')
 break
if epochs_since_lr_update >= max_epochs_wo_lr_change:
batch_lr = batch_lr -(batch_lr-lr_high_2)*0.33
epochs_since_lr_update = 0
saver_comp.restore(sess, save_path_comp)
 curr_ids = np.random.permutation(train_comp_inds.shape[0])[:100000]
train_acc = get_full_set_accuracy(sess, train_comp_inds[curr_ids], batch_size=10000)
 print('Final accuracy: train:', train_acc )<split>
ticket_counts = all['Ticket'].value_counts()
all['GrSize'] = all.apply(lambda s: ticket_counts.loc[s['Ticket']], axis=1)
 all['Cabin'].fillna('U',inplace=True)
 all['hasCabin'] = all.apply(lambda s: 0 if s['Cabin'] == 'U' else 1,axis = 1 )
def generate_val_comp_data(X, gap):
if X.shape[0] > gap:
X = X.sort_values(by=['winPlacePerc'])
 group_data = X[group_cols].values
part0 = X[['matchId', 'groupId']].values[gap:,:]
part0 = np.c_[part0, X[['matchId', 'groupId']].values[:-gap,:]]
part1 = group_data[gap:,:]
part2 = group_data[:-gap,:]
part3 = X[other_cols].values[:-gap,:]
return part0, part1, part2, part3
return np.empty(shape=(0, 4)) , np.empty(shape=(0, len(group_cols))), np.empty(shape=(0, len(group_cols))), np.empty(shape=(0, len(other_cols)))
 def get_val_batch(data, batch_size=1000, gap=1):
matchIds = data['matchId'].unique().tolist()
for i in range(int(np.ceil(len(matchIds)/batch_size))):
match_ids_batch = matchIds[i*batch_size:(i+1)*batch_size]
grouped_data = data.loc[data['matchId'].isin(match_ids_batch)].groupby('matchId' ).apply(lambda x: generate_val_comp_data(x, gap))
part0, part1, part2, part3 = list(map(np.vstack, zip(*grouped_data)))
 yield part0, part1, part2, part3<merge>
all['Fname'] = all.Name.str.extract('^ (.+?),', expand=False)
 Pas_wSib = []
all_x_0 = all[(all['SibSp'] > 0)&(all['Parch'] == 0)]
name_counts_SibSp = all_x_0['Fname'].value_counts()
for label, value in name_counts_SibSp.items() :
entries = all_x_0[all_x_0['Fname'] == label]
if(entries.shape[0] > 1 and(not(entries['Title'] == 'Mrs' ).any())) or \ 
(entries.shape[0] == 1 and entries['Title'].values[0] == 'Mrs'):
Pas_wSib.extend(entries['PassengerId'].values.tolist())
 else:
Pas_wSib.extend(\
entries[(entries['Title'] == 'Miss')|(entries['GrSize'] == 1)]['PassengerId'].values.tolist())
 Mrs_wPar = []
all_x_y = all[all['Parch'] > 0]
name_counts_Parch = all_x_y['Fname'].value_counts()
for label, value in name_counts_Parch.items() :
entries = all_x_y[all_x_y['Fname'] == label]
if entries.shape[0] == 1:
if entries['Title'].values[0] == 'Mrs' and entries['Age'].values[0] <= 30:
Mrs_wPar.extend(entries['PassengerId'].values.tolist())
 def get_features(row):
features = pd.Series(0, index = ['wSib','wSp','wCh','wPar'])
 if row['PassengerId'] in Pas_wSib:
features['wSib'] = 1
else:
if(row['SibSp'] != 0)&(row['Parch'] == 0):
features['wSp'] = 1
else:
if(( row['Title']=='Mrs')&(not row['PassengerId'] in Mrs_wPar)) | \ 
(( row['Title']=='Mr')&(not row['PassengerId'] == 680)& 
(((row['Pclass']==1)&(row['Age']>=30)) | 
(( row['Pclass']==2)&(row['Age']>=25)) | 
(( row['Pclass']==3)&(row['Age']>=20)))) :
features['wCh'] = 1
else:
features['wPar'] = 1
return features
all[['wSib','wSp','wCh','wPar']] = all.apply(lambda s: get_features(s)if s['isAlone'] == 0 else [0,0,0,0], axis = 1 )
def reorder_val(data, ids, y_comp_pred):
df = pd.DataFrame(np.c_[ids, y_comp_pred.reshape(-1,1)], columns=['matchId', 'groupId', 'ordered'])
 data = data.merge(df, on=['matchId', 'groupId'], how='left')
 data['ordered'].fillna(1, inplace=True)
 data.sort_values(by=['matchId','winPlacePerc'], inplace=True)
 zero_inds = np.nonzero(data['ordered'].values == 0)[0]
i = 0
old_order = []
new_order = []
wPP_col_num = data.columns.get_loc('winPlacePerc')
 while i < len(zero_inds):
start = zero_inds[i]
while(i+1 < len(zero_inds)) and(zero_inds[i+1] - zero_inds[i] == 1):
i += 1
end = zero_inds[i]
length = end - start + 1
i += 1
old_order.extend([i for i in range(start-1, start+length, 1)])
 new_order.extend([i for i in range(start+length-1, start-2, -1)])
 data.iloc[old_order, wPP_col_num] = data.iloc[new_order, wPP_col_num].values
print('rows swapped')
 data.drop(columns=['ordered'], inplace=True)
 return data<define_variables>
all = all.drop(['Fname','Name','Cabin','Ticket','Fare','SibSp','Parch'], axis = 1 )
train = None
train_idx = None
train_ids = None
gc.collect()<data_type_conversions>
all[all['Pclass'] == 1].groupby(['Title','isAlone','wSib','wSp','wCh','wPar'])['Survived'].agg(['count','size','mean'] )
test, test_ids = pipeline.transform(test_path)
 for num, col in enumerate(test.columns):
if col not in ['winPlacePerc', 'groupId', 'matchId']:
test[col] = test[col].astype(np.float32)
 test[col] =(( test[col] - mean_vals[col])/std_vals[col] ).astype(np.float32)
 gc.collect()<prepare_x_and_y>
all[(all['Pclass'] == 1)&(all['Title'] == 'Mr')].groupby(['hasCabin','isAlone','wSib','wSp','wCh','wPar'])['Survived'].agg(['count','size','mean'] )
y_test_dnn = pd.DataFrame(np.zeros(shape=[1,3]), columns=['matchId', 'groupId', 'winPlacePerc_pred'])
 predict_batch_size = 10000
tf.reset_default_graph()
saver = tf.train.import_meta_graph('.. /DNN_data/dnn_state.ckpt.meta')
 X = tf.get_default_graph().get_tensor_by_name('X:0')
 output = tf.get_default_graph().get_tensor_by_name('layers/output/BiasAdd:0')
 init = tf.global_variables_initializer()
with tf.Session() as sess:
init.run()
saver.restore(sess, '.. /DNN_data/dnn_state.ckpt')
 n_batches = int(np.ceil(test.shape[0]/predict_batch_size))
for batch_n in range(n_batches):
data_batch = test.iloc[batch_n*predict_batch_size:(batch_n+1)*predict_batch_size]
X_batch = data_batch.drop(columns=['matchId','groupId'])
 y_batch = data_batch[['matchId','groupId']]
y_batch['winPlacePerc_pred'] = output.eval(session=sess, feed_dict={X: X_batch})
 y_test_dnn = y_test_dnn.append(y_batch, ignore_index=True)
 y_test_dnn = y_test_dnn.iloc[1:,:]<categorify>
all[all['Pclass'] == 2].groupby(['Title','isAlone','wSib','wSp','wCh','wPar'])['Survived'].agg(['count','size','mean'] )
def rank_align_predictions(X, y, scaled=False):
X['winPlacePerc'] = y
X['rank'] = X.groupby(['matchId'])['winPlacePerc'].rank(method='dense')
 X['max_rank'] = X.groupby(['matchId'])['rank'].transform(np.max)
 adj_winPlacePerc =(X['rank'] - 1)/(X['max_rank'] - 1 + 0.0000000001)
 X.drop(columns=['winPlacePerc', 'rank', 'max_rank'], inplace=True)
 return adj_winPlacePerc
def fix_predictions(X, y, scaled=False):
y = y.copy()
y[y > 1.0] = 1.0
y[y < 0.0] = 0.0
if scaled:
max_places = X['maxPlace'].values*std_vals['maxPlace'] + mean_vals['maxPlace']
num_groups = X['numGroups'].values*std_vals['numGroups'] + mean_vals['numGroups']
else:
max_places = X['maxPlace'].values
num_groups = X['numGroups'].values
multiplier =(max_places[max_places > 1] - 1 ).astype(np.float32)
 y[max_places > 1] = np.round(y[max_places > 1] * multiplier)/ multiplier
y[max_places == 1] = 1.0
y[max_places <= 0] = 0.0
y[num_groups <= 1] = 0.0
return y<merge>
all[all['Pclass'] == 3].groupby(['Title','isAlone','wSib','wSp','wCh','wPar'])['Survived'].agg(['count','size','mean'] )
y_test_dnn['winPlacePerc_pred'] = rank_align_predictions(y_test_dnn.drop(columns=['winPlacePerc_pred']), y_test_dnn['winPlacePerc_pred'])
 y_test_dnn = y_test_dnn.merge(test[['matchId', 'groupId', 'numGroups', 'maxPlace']], on=['matchId', 'groupId'])
 y_test_dnn['winPlacePerc_pred'] = fix_predictions(y_test_dnn.drop(columns=['winPlacePerc_pred']), y_test_dnn['winPlacePerc_pred'], scaled=True )<merge>
all[(all['Pclass'] == 3)&(all['Title'] != 'Mr')].groupby(['Title','FamSize'])['Survived'].agg(['count','size','mean'] )
test = test.merge(y_test_dnn[['matchId', 'groupId', 'winPlacePerc_pred']], on=['matchId', 'groupId'])
 test.rename(columns={'winPlacePerc_pred':'winPlacePerc'}, inplace=True )<prepare_x_and_y>
all[(all['Pclass'] == 3)&(all['Title'] != 'Mr')].groupby(['Title','FamSizeBin','isAlone','wSib','wSp','wCh','wPar'])['Survived'].agg(['count','size','mean'] )
n_epochs = 6
batch_size = 10000
comp_thresh = [-0.1, 0, -0.1]
y_comp_diff_data = {}
y_comp_pred_data = {}
tf.reset_default_graph()
saver_comp = tf.train.import_meta_graph('.. /DNN_data/dnn_state_comp.ckpt.meta')
 X1 = tf.get_default_graph().get_tensor_by_name('X1:0')
 X2 = tf.get_default_graph().get_tensor_by_name('X2:0')
 X3 = tf.get_default_graph().get_tensor_by_name('X3:0')
 output = tf.get_default_graph().get_tensor_by_name('layers/output:0')
 init = tf.global_variables_initializer()
with tf.Session() as sess:
saver_comp.restore(sess, save_path_comp)
 for epoch in range(n_epochs):
y_comp_pred = np.empty(shape=[0])
 y_comp_diff = np.empty(shape=[0])
 all_ids = np.empty(shape=[0, 2])
 for batch_ids, X1_batch, X2_batch, X3_batch in get_val_batch(test, batch_size):
feed_dict = {X1:X1_batch, X2:X2_batch, X3:X3_batch}
batch_output = output.eval(feed_dict=feed_dict ).reshape(-1)
 assert len(batch_output)% 2 ==0
y_batch_pred = batch_output[:int(len(batch_output)/2)]
y_batch_comp = batch_output[:int(len(batch_output)/2)] - batch_output[int(len(batch_output)/2):]
y_comp_pred = np.r_[y_comp_pred, y_batch_pred]
y_comp_diff = np.r_[y_comp_diff, y_batch_comp]
all_ids = np.r_[all_ids, batch_ids[:,:2]]
y_comp_pred_data[epoch] = y_comp_pred
y_comp_diff_data[epoch] = y_comp_diff
thresh_ind = np.minimum(len(comp_thresh)-1, epoch)
 y_comp_diff = y_comp_diff > comp_thresh[thresh_ind]
print('epoch:', epoch, 'number of zeros:', len(np.nonzero(y_comp_diff == 0)[0]))
if np.min(y_comp_diff)== 0:
test = reorder_val(test, all_ids, y_comp_diff)
 else:
break<merge>
def get_survived_1(row):
if row['Pclass'] in [1,2]:
if row['Title'] == 'Mr':
survived = 0
else:
survived = 1
else:
if row['Title'] == 'Mr' or row['FamSizeBin'] == 1:
survived = 0
else:
survived = 1
return survived
def create_submission_table(X, y, id_table):
out = X[['matchId', 'groupId']]
out['winPlacePerc'] = y
out = id_table.merge(out, on=['matchId', 'groupId'])
 out = out.drop(columns=['groupId', 'matchId'])
 return out<save_to_csv>
X_train = all.iloc[:891,:]
X_test = all.iloc[891:,:]
y_train = all.iloc[:891,:]['Survived']
y_train_hat = X_train.apply(lambda s: get_survived_1(s), axis = 1)
 predictions = pd.DataFrame({'PassengerId': test['PassengerId'], 'Survived': 0})
 predictions['Survived'] = X_test.apply(lambda s: get_survived_1(s), axis = 1)
 predictions.to_csv('submission-1.csv', index=False)
 score = metrics.accuracy_score(y_train_hat, y_train)
 print('Train Accuracy: {}'.format(score))
submission = create_submission_table(test, test['winPlacePerc'], test_ids)
 submission.to_csv('submission.csv', index=False)
 submission.head(50 )<import_modules>
all[(all['Pclass'] == 3)&(all['Title'] != 'Mr')&(all['FamSizeBin'] == 0)].groupby(['Title','Embarked'])['Survived'].agg(['count','size','mean'] )
import gc
import sys
import numpy as np
import pandas as pd<categorify>
def get_survived_2(row):
if row['Pclass'] in [1,2]:
if row['Title'] == 'Mr':
survived = 0
else:
survived = 1
else:
if row['Title'] == 'Mr' or row['FamSizeBin'] == 1 or(row['Title'] == 'Miss' and row['Embarked'] == 'S'):
survived = 0
else:
survived = 1
return survived
def df_footprint_reduce(df, skip_obj=False, skip_int=False, skip_float=False, print_comparison=True):

 if print_comparison:
print(f"Dataframe size before shrinking column types into smallest possible: {round(( sys.getsizeof(df)/1024/1024),4)} MB")
 for column in df.columns:
if(skip_obj is False)and(str(df[column].dtype)[:6] == 'object'):
num_unique_values = len(df[column].unique())
 num_total_values = len(df[column])
 if num_unique_values / num_total_values < 0.5:
df.loc[:,column] = df[column].astype('category')
 else:
df.loc[:,column] = df[column]
elif(skip_int is False)and(str(df[column].dtype)[:3] == 'int'):
if df[column].min() > np.iinfo(np.int8 ).min and df[column].max() < np.iinfo(np.int8 ).max:
df[column] = df[column].astype(np.int8)
 elif df[column].min() > np.iinfo(np.int16 ).min and df[column].max() < np.iinfo(np.int16 ).max:
df[column] = df[column].astype(np.int16)
 elif df[column].min() > np.iinfo(np.int32 ).min and df[column].max() < np.iinfo(np.int32 ).max:
df[column] = df[column].astype(np.int32)
 elif(skip_float is False)and(str(df[column].dtype)[:5] == 'float'):
if df[column].min() > np.finfo(np.float16 ).min and df[column].max() < np.finfo(np.float16 ).max:
df[column] = df[column].astype(np.float16)
 elif df[column].min() > np.finfo(np.float32 ).min and df[column].max() < np.finfo(np.float32 ).max:
df[column] = df[column].astype(np.float32)
 if print_comparison:
print(f"Dataframe size after shrinking column types into smallest possible: {round(( sys.getsizeof(df)/1024/1024),4)} MB")
 return df<feature_engineering>
y_train_hat = X_train.apply(lambda s: get_survived_2(s), axis = 1)
 predictions['Survived'] = X_test.apply(lambda s: get_survived_2(s), axis = 1)
 predictions.to_csv('submission-2.csv', index=False)
 score = metrics.accuracy_score(y_train_hat, y_train)
 print('Train Accuracy: {}'.format(score))
def feature_engineering(df,is_train=True):
if is_train:
df = df[df['maxPlace'] > 1].copy()
target = 'winPlacePerc'
print('Grouping similar match types together')
 df.loc[(df['matchType'] == 'solo'), 'matchType'] = 1
df.loc[(df['matchType'] == 'normal-solo'), 'matchType'] = 1
df.loc[(df['matchType'] == 'solo-fpp'), 'matchType'] = 1
df.loc[(df['matchType'] == 'normal-solo-fpp'), 'matchType'] = 1
df.loc[(df['matchType'] == 'duo'), 'matchType'] = 2
df.loc[(df['matchType'] == 'normal-duo'), 'matchType'] = 2
df.loc[(df['matchType'] == 'duo-fpp'), 'matchType'] = 2
df.loc[(df['matchType'] == 'normal-duo-fpp'), 'matchType'] = 2
df.loc[(df['matchType'] == 'squad'), 'matchType'] = 3
df.loc[(df['matchType'] == 'normal-squad'), 'matchType'] = 3
df.loc[(df['matchType'] == 'squad-fpp'), 'matchType'] = 3
df.loc[(df['matchType'] == 'normal-squad-fpp'), 'matchType'] = 3
df.loc[(df['matchType'] == 'flaretpp'), 'matchType'] = 0
df.loc[(df['matchType'] == 'flarefpp'), 'matchType'] = 0
df.loc[(df['matchType'] == 'crashtpp'), 'matchType'] = 0
df.loc[(df['matchType'] == 'crashfpp'), 'matchType'] = 0
df.loc[(df['rankPoints'] < 0), 'rankPoints'] = 0
print('Adding new features using existing ones')
 df['headshotrate'] = df['kills']/df['headshotKills']
df['killStreakrate'] = df['killStreaks']/df['kills']
df['healthitems'] = df['heals'] + df['boosts']
df['totalDistance'] = df['rideDistance'] + df["walkDistance"] + df["swimDistance"]
df['killPlace_over_maxPlace'] = df['killPlace'] / df['maxPlace']
df['headshotKills_over_kills'] = df['headshotKills'] / df['kills']
df['distance_over_weapons'] = df['totalDistance'] / df['weaponsAcquired']
df['walkDistance_over_heals'] = df['walkDistance'] / df['heals']
df['walkDistance_over_kills'] = df['walkDistance'] / df['kills']
df['killsPerWalkDistance'] = df['kills'] / df['walkDistance']
df['skill'] = df['headshotKills'] + df['roadKills']
print('Adding normalized features')
 df['playersJoined'] = df.groupby('matchId')['matchId'].transform('count')
 gc.collect()
df['killsNorm'] = df['kills']*(( 100-df['playersJoined'])/100 + 1)
 df['damageDealtNorm'] = df['damageDealt']*(( 100-df['playersJoined'])/100 + 1)
 df['maxPlaceNorm'] = df['maxPlace']*(( 100-df['playersJoined'])/100 + 1)
 df['matchDurationNorm'] = df['matchDuration']*(( 100-df['playersJoined'])/100 + 1)
 df['headshotKillsNorm'] = df['headshotKills']*(( 100-df['playersJoined'])/100 + 1)
 df['killPlaceNorm'] = df['killPlace']*(( 100-df['playersJoined'])/100 + 1)
 df['killPointsNorm'] = df['killPoints']*(( 100-df['playersJoined'])/100 + 1)
 df['killStreaksNorm'] = df['killStreaks']*(( 100-df['playersJoined'])/100 + 1)
 df['longestKillNorm'] = df['longestKill']*(( 100-df['playersJoined'])/100 + 1)
 df['roadKillsNorm'] = df['roadKills']*(( 100-df['playersJoined'])/100 + 1)
 df['teamKillsNorm'] = df['teamKills']*(( 100-df['playersJoined'])/100 + 1)
 df['damageDealtNorm'] = df['damageDealt']*(( 100-df['playersJoined'])/100 + 1)
 df['DBNOsNorm'] = df['DBNOs']*(( 100-df['playersJoined'])/100 + 1)
 df['revivesNorm'] = df['revives']*(( 100-df['playersJoined'])/100 + 1)
 df = df_null_cleaner(df,fill_with=0)
 features = list(df.columns)
 features.remove("Id")
 features.remove("matchId")
 features.remove("groupId")
 features.remove("matchType")
 features.remove("maxPlace")
 y = pd.DataFrame()
if is_train:
print('Preparing target variable')
 y = df.groupby(['matchId','groupId'])[target].agg('mean')
 gc.collect()
features.remove(target)
 print('Aggregating means')
 means_features = list(df.columns)
 means_features.remove("Id")
 means_features.remove("matchId")
 means_features.remove("groupId")
 means_features.remove("matchType")
 means_features.remove("maxPlace")
 means_features.remove("playersJoined")
 means_features.remove("matchDuration")
 means_features.remove("numGroups")
 means_features.remove("teamKillsNorm")
 if is_train:
means_features.remove(target)
 agg = df.groupby(['matchId','groupId'])[means_features].agg('mean')
 gc.collect()
agg_rank = agg.groupby('matchId')[means_features].rank(pct=True ).reset_index()
gc.collect()
if is_train:
X = agg.reset_index() [['matchId','groupId']]
else:
X = df[['matchId','groupId']]
X = X.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 X = X.merge(agg_rank, suffixes=["_mean", "_mean_rank"], how='left', on=['matchId', 'groupId'])
 del agg, agg_rank
gc.collect()
print('Aggregating maxes')
 maxes_features = list(df.columns)
 maxes_features.remove("Id")
 maxes_features.remove("matchId")
 maxes_features.remove("groupId")
 maxes_features.remove("matchType")
 maxes_features.remove("DBNOsNorm")
 maxes_features.remove("damageDealtNorm")
 maxes_features.remove("headshotKillsNorm")
 maxes_features.remove("killPlaceNorm")
 maxes_features.remove("killPlace_over_maxPlace")
 maxes_features.remove("killPointsNorm")
 maxes_features.remove("killStreaksNorm")
 maxes_features.remove("killsNorm")
 maxes_features.remove("longestKillNorm")
 maxes_features.remove("matchDurationNorm")
 maxes_features.remove("matchDuration")
 maxes_features.remove("maxPlaceNorm")
 maxes_features.remove("maxPlace")
 maxes_features.remove("numGroups")
 maxes_features.remove("playersJoined")
 maxes_features.remove("revivesNorm")
 maxes_features.remove("roadKillsNorm")
 maxes_features.remove("teamKillsNorm")
 if is_train:
maxes_features.remove(target)
 agg = df.groupby(['matchId','groupId'])[maxes_features].agg('max')
 gc.collect()
agg_rank = agg.groupby('matchId')[maxes_features].rank(pct=True ).reset_index()
gc.collect()
X = X.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 X = X.merge(agg_rank, suffixes=["_max", "_max_rank"], how='left', on=['matchId', 'groupId'])
 del agg, agg_rank
gc.collect()
print('Aggregating mins')
 mins_features = list(df.columns)
 mins_features.remove("Id")
 mins_features.remove("matchId")
 mins_features.remove("groupId")
 mins_features.remove("matchType")
 mins_features.remove("DBNOsNorm")
 mins_features.remove("damageDealtNorm")
 mins_features.remove("headshotKillsNorm")
 mins_features.remove("killPlaceNorm")
 mins_features.remove("killPlace_over_maxPlace")
 mins_features.remove("killPointsNorm")
 mins_features.remove("killStreaksNorm")
 mins_features.remove("killsNorm")
 mins_features.remove("longestKillNorm")
 mins_features.remove("matchDurationNorm")
 mins_features.remove("matchDuration")
 mins_features.remove("maxPlaceNorm")
 mins_features.remove("maxPlace")
 mins_features.remove("numGroups")
 mins_features.remove("playersJoined")
 mins_features.remove("revivesNorm")
 mins_features.remove("roadKillsNorm")
 mins_features.remove("teamKillsNorm")
 if is_train:
mins_features.remove(target)
 agg = df.groupby(['matchId','groupId'])[mins_features].agg('min')
 gc.collect()
agg_rank = agg.groupby('matchId')[mins_features].rank(pct=True ).reset_index()
gc.collect()
X = X.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 X = X.merge(agg_rank, suffixes=["_min", "_min_rank"], how='left', on=['matchId', 'groupId'])
 del agg, agg_rank
gc.collect()
print('Aggregating group sizes')
 agg = df.groupby(['matchId','groupId'] ).size().reset_index(name='group_size')
 gc.collect()
X = X.merge(agg, how='left', on=['matchId', 'groupId'])
 print('Aggregating match means')
 agg = df.groupby(['matchId'])[features].agg('mean' ).reset_index()
gc.collect()
X = X.merge(agg, suffixes=["", "_match_mean"], how='left', on=['matchId'])
 print('Aggregating match sizes')
 agg = df.groupby(['matchId'] ).size().reset_index(name='match_size')
 gc.collect()
X = X.merge(agg, how='left', on=['matchId'])
 del df, agg
gc.collect()
X.drop(columns = ['matchId',
'groupId'
], axis=1, inplace=True)
 gc.collect()
if is_train:
return X, y
return X<load_from_csv>
all[(all['Pclass'] == 3)&(all['Title'] == 'Miss')&(all['FamSizeBin'] == 0)].groupby(['Title','wPar','Embarked'])['Survived'].agg(['count','size','mean'] )
X_train = pd.read_csv('.. /input/train_V2.csv', engine='c' )<concatenate>
def get_survived_3(row):
if row['Pclass'] in [1,2]:
if row['Title'] == 'Mr':
survived = 0
else:
survived = 1
else:
if row['Title'] == 'Mr' or row['FamSizeBin'] == 1 or \ 
(row['Title'] == 'Miss' and row['Embarked'] == 'S' and row['wPar'] == 0):
survived = 0
else:
survived = 1
return survived
X_train = df_footprint_reduce(X_train, skip_obj=True)
 gc.collect()<feature_engineering>
y_train_hat = X_train.apply(lambda s: get_survived_3(s), axis = 1)
 predictions['Survived'] = X_test.apply(lambda s: get_survived_3(s), axis = 1)
 predictions.to_csv('submission-3.csv', index=False)
 score = metrics.accuracy_score(y_train_hat, y_train)
 print('Train Accuracy: {}'.format(score))
X_train, y_train = feature_engineering(X_train, True)
 gc.collect()<concatenate>
all['Sex'] = all['Sex'].map({'male': 0, 'female': 1} ).astype(int)
 all['Embarked'].fillna(all['Embarked'].value_counts().index[0], inplace=True)
 all_dummies = pd.get_dummies(all, columns = ['Title','Pclass','Embarked'],\
prefix=['Title','Pclass','Embarked'], drop_first = True)
 all_dummies = all_dummies.drop(['PassengerId','Survived'], axis = 1 )
X_train = df_footprint_reduce(X_train, skip_obj=True)
 gc.collect()<import_modules>
all_dummies_i = pd.DataFrame(data=KNN(k=3, verbose = False ).fit_transform(all_dummies ).astype(int),
columns=all_dummies.columns, index=all_dummies.index )
from sklearn.model_selection import train_test_split, GridSearchCV<split>
all_dummies_i['isAlwSib'] = all_dummies_i.apply(lambda s: 1 if(s['isAlone'] == 1)|(s['wSib'] == 1)else 0 ,axis = 1)
 all_dummies_i = all_dummies_i.drop(['isAlone','wSib','Sex','GrSize'], axis = 1 )
X_train, X_validation, y_train, y_validation = train_test_split(X_train,
y_train,
test_size=0.2)
 gc.collect()<import_modules>
X_train = all_dummies_i.iloc[:891,:]
X_test = all_dummies_i.iloc[891:,:]
import lightgbm as lgb<train_model>
scaler = StandardScaler()
scaler.fit(X_train[['Age']])
 X_train['Age'] = scaler.transform(X_train[['Age']])
 X_test['Age'] = scaler.transform(X_test[['Age']] )
parameters = { 'objective': 'regression_l1',
'learning_rate': 0.01
}<train_on_grid>
cv = RepeatedStratifiedKFold(n_splits=5, n_repeats=10, random_state=1 )
def find_best_hyperparameters(model):
gridParams = {
'learning_rate' : [0.1, 0.01 , 0.05],
'n_estimators ' : [1000, 10000, 20000],
'bagging_fraction' : [0.5, 0.6 ,0.7],
'feature_fraction' : [0.5, 0.6 ,0.7],
'num_leaves' : [31, 80, 140]
}
grid = GridSearchCV(model,
gridParams,
verbose=5,
cv=3)
 grid.fit(X_train, y_train)
 print('Best parameters: %s' % grid.best_params_)
 print('Accuracy: %.2f' % grid.best_score_)
 return<create_dataframe>
svm_grid = {'C': [10,11,12,13,14,15,16,17,18,19,20], 'gamma': ['auto']}
svm_search = GridSearchCV(estimator = SVC() , param_grid = svm_grid, cv = cv, refit=True, n_jobs=1 )
X_train = lgb.Dataset(X_train, label=y_train)
 X_validation = lgb.Dataset(X_validation, label=y_validation)
 gc.collect()<train_model>
svm_search.fit(X_train, train['Survived'])
 svm_best = svm_search.best_estimator_
print("Cross-validation accuracy: {}, standard deviation: {}, with parameters {}" 
.format(svm_search.best_score_, svm_search.cv_results_['std_test_score'][svm_search.best_index_],
svm_search.best_params_))
%%time
model = lgb.train(parameters,
X_train,
num_boost_round = 40000,
valid_sets=[X_validation,X_train] )<compute_test_metric>
y_train_hat = svm_best.predict(X_train)
 print('Train Accuracy: {}' 
.format(metrics.accuracy_score(y_train_hat, y_train)))
 predictions['Survived'] = svm_best.predict(X_test)
 predictions.to_csv('submission-svm.csv', index=False )
<import_modules><EOS> 
def get_survived_svm_rule(row):
if row['Pclass'] in [1,2]:
if row['Title'] == 'Mr':
survived = 0
else:
survived = 1
else:
if row['Title'] == 'Mr' or row['FamSizeBin'] == 1 or \ 
(row['Title'] == 'Miss' and row['Embarked'] == 'S' and row['Age'] >= 18):
survived = 0
else:
survived = 1
return survived