Kaggle competition: PUBG Finish Placement Prediction (Kernels Only)
<SOS> metric: MAE Kaggle data source: pubg-finish-placement-prediction-(kernels-only)<groupby>
kernel_start_time = datetime.datetime.now()
train_path = '.. /input/train_V2.csv'
test_path = '.. /input/test_V2.csv'
print(os.listdir(".. /input"))
data_all.groupby(['Pclass', 'Sex'])['Age'].mean()<filter>
data_types = {'Id':str,'groupId':str,'matchId':str,'assists':np.int8,'boosts':np.int8,'damageDealt':np.float16,'DBNOs':np.int8,
'headshotKills':np.int8,'heals':np.int8,'killPlace':np.int8,'killPoints':np.int16,'kills':np.int8,'killStreaks':np.int8,'longestKill':np.float16,
'matchDuration':np.int16,'matchType':str,'maxPlace':np.int8,'numGroups':np.int8,'rankPoints':np.int16,'revives':np.int8,'rideDistance':np.float16,
'roadKills':np.int8,'swimDistance':np.float16,'teamKills':np.int8,'vehicleDestroys':np.int8,'walkDistance':np.float16,'weaponsAcquired':np.int16,
'winPoints':np.int16,'winPlacePerc':np.float16}
data_types['walkDistance'] = np.float32
data_types['rideDistance'] = np.float32
data_types['damageDealt'] = np.float32
index = data_all['Age'].index[data_all['Age'].apply(np.isnan)]
data_all.loc[index, ['Pclass', 'Sex', 'Age']]<feature_engineering>
zero_stdev_cols = ['median_match_roadKills', 'median_match_vehicleDestroys', 'median_match_road_kills_per_rideDistance', 'std_match_longestKill']
ulesess_stat_cols = ['median_match_revives','median_match_teamKills','median_match_swimDistance_norm',
'max_match_kill_streak_rate','max_group_roadKills','min_group_roadKills','median_match_assists',
'sum_group_roadKills','sum_group_vehicleDestroys','min_group_vehicleDestroys']
useless_match_type_names = ['normal-duo','flarefpp','normal-solo','crashtpp','normal-squad','flaretpp','normal-duo-fpp',
'normal-squadfpp', 'normal-squad-fpp', 'crashfpp','duo','squad-fpp','normal-solo-fpp']
all_useless_cols = zero_stdev_cols + ulesess_stat_cols + useless_match_type_names
all_useless_cols = set(all_useless_cols)
 min_match_useful_cols = ['min_match_rankPoints', 'min_match_winPoints', 'min_match_killPoints']
data_all['Age'] = data_all.groupby(['Pclass', 'Sex'])['Age'].apply(lambda x: x.fillna(x.mean()))
train_data, test_data = divide_data(data_all)
 data_dict['train data'] = train_data
data_dict['test data'] = test_data
data_all.loc[index, ['Pclass', 'Sex', 'Age']]<feature_engineering>
def fix_missing_ranks(X, mean_ranks=None, rank_stds=None, rank_cols=['rankPoints', 'winPoints']):
if(mean_ranks is None)or(rank_stds is None):
mean_ranks = {}
rank_stds = {}
for rank_col in rank_cols:
mean_ranks[rank_col] = X.loc[X[rank_col] > 1, rank_col].mean()
rank_stds[rank_col] = X.loc[X[rank_col] > 1, rank_col].std()
rank_deltas =(X.loc[(X['rankPoints'] > 1)&(X['winPoints'] <= 1), 'rankPoints'] - mean_ranks['rankPoints'])/ rank_stds['rankPoints']
X.loc[(X['rankPoints'] > 1)&(X['winPoints'] <= 1), 'winPoints'] = mean_ranks['winPoints'] + rank_stds['winPoints']*rank_deltas
win_deltas =(X.loc[(X['rankPoints'] <= 1)&(X['winPoints'] > 1), 'winPoints'] - mean_ranks['winPoints'])/ rank_stds['winPoints']
X.loc[(X['rankPoints'] <= 1)&(X['winPoints'] > 1), 'rankPoints'] = mean_ranks['rankPoints'] + rank_stds['rankPoints']*win_deltas
X.loc[(X['rankPoints'] <= 1)&(X['winPoints'] <= 1), 'rankPoints'] = mean_ranks['rankPoints']
X.loc[(X['rankPoints'] <= 1)&(X['winPoints'] <= 1), 'winPoints'] = mean_ranks['winPoints']
return X, mean_ranks, rank_stds
data_all.Cabin = data_all.Cabin.fillna('Unknown')
 data_all['Deck'] = data_all['Cabin'].str[0]<count_values>
def add_player_features(X):
X['headshot_rate'] = X['headshotKills'] /(X['kills'] + 0.00001)
 X['kill_streak_rate'] = X['killStreaks'] /(X['kills'] + 0.00001)
 X['kills_assists'] = X['kills'] + X['assists']
X['heals_boosts'] = X['heals'] + X['boosts']
X['total_distance'] = X['walkDistance'] + X['rideDistance'] + X['swimDistance']
X['kills_assists_per_heal_boost'] = X['kills_assists'] /(X['heals_boosts'] + 1)
 X['damageDealt_per_heal_boost'] = X['damageDealt'] /(X['heals_boosts'] + 1)
 X['road_kills_per_rideDistance'] = X['roadKills'] /(X['rideDistance'] + 0.01)
 X['maxPlace_per_numGroups'] = X['maxPlace'] / X['numGroups']
X['assists_per_kill'] = X['assists'] /(X['kills'] + X['assists'] + 0.0001)
 X['killPlace'] = X['killPlace'] - 1
return X
data_all.groupby(['Pclass'] ).Deck.value_counts()<drop_column>
def create_basic_group_info(X):
group_cols = ['matchId', 'groupId', 'matchDuration', 'matchType', 'maxPlace', 'numGroups', 'maxPlace_per_numGroups', 'winPlacePerc', 'killPlace']
if 'winPlacePerc' not in X.columns:
group_cols.remove('winPlacePerc')
 pl_data_grouped = X[group_cols].groupby(['matchId', 'groupId'])
 gr_data = pl_data_grouped.first()
gr_data.drop(columns='killPlace', inplace=True)
 gr_data['raw_groupSize'] = pl_data_grouped['numGroups'].count()
gr_data['groupSize'] = gr_data['raw_groupSize']
gr_data['group_size_overflow'] =(gr_data['groupSize'] > 4 ).astype(np.int8)
 gr_data.loc[gr_data['groupSize'] > 4, ['groupSize']] = 2
gr_data['meanGroupSize'] = gr_data.groupby('matchId')['groupSize'].transform(np.mean)
 gr_data['medianGroupSize'] = gr_data.groupby('matchId')['groupSize'].transform(np.median)
 gr_data['maxKillPlace'] = pl_data_grouped['killPlace'].max().groupby('matchId' ).transform(np.max)
 gr_data['totalPlayers'] = gr_data.groupby('matchId')['groupSize'].transform(sum)
 gr_data['totalPlayersAdjusted'] = gr_data['maxPlace'].astype(float)* gr_data['totalPlayers'] /(gr_data['numGroups'] + 0.01)
 gr_data['totalPlayersAdjusted'] = gr_data['totalPlayersAdjusted'].apply(lambda x: np.minimum(100.0, x))
gr_data['num_opponents'] = gr_data['totalPlayersAdjusted'] - gr_data['groupSize']
X = X.merge(gr_data[['num_opponents', 'totalPlayersAdjusted', 'groupSize', 'raw_groupSize', 'maxKillPlace']], on=['matchId', 'groupId'])
 print('group size counts:')
 print(X['raw_groupSize'].value_counts())
 X['revives_per_groupSize'] = X['revives'] /(X['groupSize'] - 1 + 0.001)
 X['kills_assists_norm_both'] = X['kills_assists'].astype(np.float32)/ X['num_opponents'] / X['matchDuration']
X['killPlace_norm'] = X['killPlace']/(X['maxKillPlace'] + 0.000001)
 X['damageDealt_norm_both'] = X['damageDealt'].astype(np.float32)/ X['num_opponents'] / X['matchDuration']
X['DBNOs_norm'] = X['DBNOs'].astype(np.float32)/ X['num_opponents'] / X['matchDuration']
X['heals_norm'] = X['heals'].astype(np.float32)/ X['matchDuration']
X['boosts_norm'] = X['boosts'].astype(np.float32)/ X['matchDuration']
X['walkDistance_norm'] = X['walkDistance'].astype(np.float32)/ X['matchDuration']
X['rideDistance_norm'] = X['rideDistance'].astype(np.float32)/ X['matchDuration']
X['swimDistance_norm'] = X['swimDistance'].astype(np.float32)/ X['matchDuration']
gr_data = reduce_mem_usage(gr_data)
 gr_data.drop(columns=list(set(gr_data.columns)&all_useless_cols))
return gr_data, X
data_all.groupby(['Deck'] ).mean().drop(['PassengerId', 'SibSp', 'Parch'], axis=1 )<drop_column>
def create_group_and_match_stats(data, gr_data):
group_stats_cols = ['assists','boosts','DBNOs','killPoints','longestKill','rankPoints', 'road_kills_per_rideDistance',
'kills_assists_norm_both', 'damageDealt_norm_both', 'DBNOs_norm', 'heals_boosts', 'assists_per_kill', 'killPlace_norm',
'revives','roadKills','teamKills','vehicleDestroys','weaponsAcquired','winPoints','headshot_rate','kill_streak_rate',
'kills_assists', 'heals_norm','walkDistance_norm','rideDistance_norm','swimDistance_norm',
'damageDealt_per_heal_boost','kills_assists_per_heal_boost']
match_stats_cols = ['assists','boosts','DBNOs','killPoints','longestKill','rankPoints', 'road_kills_per_rideDistance',
'kills_assists_norm_both', 'damageDealt_norm_both', 'DBNOs_norm', 'heals_boosts', 'assists_per_kill',
'revives','roadKills','teamKills','vehicleDestroys','weaponsAcquired','winPoints','headshot_rate','kill_streak_rate',
'kills_assists', 'heals_norm','walkDistance_norm','rideDistance_norm','swimDistance_norm',
'damageDealt_per_heal_boost','kills_assists_per_heal_boost']
pl_data_grouped_by_group = data.groupby(['matchId', 'groupId'])
 pl_data_grouped_by_match = data.groupby(['matchId'])
 group_funcs = {'min':np.min, 'max':np.max, 'sum':np.sum, 'median': np.mean}
match_funcs = {'min':np.min,'max':np.max, 'sum':np.sum, 'median': np.median, 'std':np.std}
extra_group_stats = pl_data_grouped_by_group[['matchId', 'groupId']].first().reset_index(drop=True)
 extra_match_stats = pl_data_grouped_by_match[['matchId']].first().reset_index(drop=True)
 for colname in group_stats_cols:
for f_name, func in group_funcs.items() :
gr_col_name = f_name + '_group_' + colname
if(gr_col_name not in all_useless_cols)or(( gr_col_name + '_rank')not in all_useless_cols):
if func is not sum:
extra_group_stats[gr_col_name] = pl_data_grouped_by_group[colname].agg(func ).values
else:
extra_group_stats[gr_col_name] = pl_data_grouped_by_group[colname].agg(func ).values * sum_multipliers
for colname in match_stats_cols:
for f_name, func in match_funcs.items() :
m_col_name = f_name + '_match_' + colname
if m_col_name not in all_useless_cols:
if func is np.std:
extra_match_stats[m_col_name] = pl_data_grouped_by_match[colname].agg(func ).fillna(0 ).values
elif func is np.min:
if m_col_name in min_match_useful_cols:
extra_match_stats[m_col_name] = pl_data_grouped_by_match[colname].agg(func ).values
else:
extra_match_stats[m_col_name] = pl_data_grouped_by_match[colname].agg(func ).values
extra_group_stats.set_index(['matchId', 'groupId'], inplace=True)
 extra_match_stats.set_index(['matchId'], inplace=True)
 pl_data_grouped_by_group = None
pl_data_grouped_by_match = None
select_cols = []
for col in extra_group_stats.columns:
if(( col + '_rank')not in all_useless_cols)and(col not in ['matchId', 'groupId']):
select_cols.append(col)
 rank_data = extra_group_stats.groupby(['matchId'])
 rank_data = rank_data[select_cols].rank() - 1
gc.collect()
max_rank_data = rank_data.groupby(['matchId'] ).transform(np.max)
 rank_data = rank_data /(max_rank_data + 0.0001)
 max_rank_data = None
gc.collect()
print('rank data created')
 gr_col_to_drop = list(set(extra_group_stats.columns)& all_useless_cols)
 extra_group_stats.drop(columns=gr_col_to_drop, inplace=True)
 gc.collect()
extra_group_stats = extra_group_stats.join(rank_data, on=['matchId', 'groupId'], rsuffix='_rank')
 extra_group_stats.reset_index(level=1, inplace=True)
 rank_data = None
gc.collect()
print('rank data merged')
 extra_group_stats = reduce_mem_usage(extra_group_stats)
 extra_match_stats = reduce_mem_usage(extra_match_stats)
 merged_features = extra_group_stats.merge(extra_match_stats, on=['matchId'])
 extra_group_stats = None
extra_match_stats = None
gc.collect()
print('extra match and group stats merged')
 merged_features = merged_features.merge(gr_data, on=['matchId', 'groupId'])
 gr_data = None
gc.collect()
print('group data and stats merged')
 cats = merged_features['matchType'].unique()
cats = set(cats)- set(all_useless_cols)
 encoded_data = np.empty(shape=(merged_features.shape[0], 0), dtype=np.int8)
 for category in cats:
encoded_data = np.c_[encoded_data,(merged_features[['matchType']] == category ).values.reshape(-1,1 ).astype(np.int8)]
encoded_data = pd.DataFrame(encoded_data, columns=cats, index=merged_features.index, dtype=np.int8)
 print('matchType data created')
 for col in encoded_data.columns:
merged_features[col] = encoded_data[col]
encoded_data = None
gc.collect()
print('match type data merged')
 cols_to_drop = ['matchType']
merged_features = merged_features.drop(columns=cols_to_drop)
 return merged_features
deck_avg_fare = data_all.groupby(['Deck'] ).Fare.mean().drop('U')
 deck_avg_fare<feature_engineering>
def remove_outliers(X):
outliers =(X['walkDistance'] > 10000)|(X['rideDistance'] > 15000)|(X['swimDistance'] > 1000)|(( X['kills'] > 0)&(X['total_distance'] == 0))
outliers = outliers |(X['kills'] > 30)|(X['longestKill'] > 800)|(X['weaponsAcquired'] > 40)
 X = X.loc[~outliers]
outlier_data = X.loc[outliers]
return X, outlier_data
indices_U = data_all[data_all['Deck'] == 'U'].index
for i in indices_U:
fare = data_all.iloc[i].Fare
nearest_avg_fare = min(deck_avg_fare, key=lambda x:abs(x-fare))
deck = deck_avg_fare[deck_avg_fare == nearest_avg_fare].index[0]
data_all['Deck'].iloc[i] = deck
train_data, test_data = divide_data(data_all)
 data_dict['train data'] = train_data
data_dict['test data'] = test_data<filter>
class DataPipeline() :
def __init__(self, pipeline=None):
if pipeline is not None:
self.rank_means = pipeline.rank_means
self.rank_stds = pipeline.rank_stds
return
def fit_transform(self, data_path):
data = pd.read_csv(data_path)
 for col_name, col_type in data_types.items() :
data[col_name] = data[col_name].astype(col_type)
 data = data.loc[~data['winPlacePerc'].isna() ]
data = data.loc[data['maxPlace'] > 1]
data, self.rank_means, self.rank_stds = fix_missing_ranks(data)
 data = add_player_features(data)
 gr_data, data = create_basic_group_info(data)
 cols_to_drop = ['kills', 'headshotKills', 'killStreaks', 'walkDistance', 'rideDistance', 'swimDistance', 'heals']
data.drop(columns=cols_to_drop, inplace=True)
 data_ids = data[['Id', 'matchId', 'groupId']]
merged_features = create_group_and_match_stats(data, gr_data)
 gc.collect()
print('final feature dataframe shape:', merged_features.shape)
 return merged_features.reset_index() , data_ids
def transform(self, data_path):
data = pd.read_csv(data_path)
 for col_name, col_type in data_types.items() :
if col_name != 'winPlacePerc':
data[col_name] = data[col_name].astype(col_type)
 data = fix_missing_ranks(data, self.rank_means, self.rank_stds)[0]
data = add_player_features(data)
 gr_data, data = create_basic_group_info(data)
 cols_to_drop = ['kills', 'headshotKills', 'killStreaks', 'walkDistance', 'rideDistance', 'swimDistance', 'heals']
data.drop(columns=cols_to_drop, inplace=True)
 data_ids = data[['Id', 'matchId', 'groupId']]
merged_features = create_group_and_match_stats(data, gr_data)
 print('final feature dataframe shape:', merged_features.shape)
 return merged_features.reset_index() , data_ids
data_all.loc[indices_U, ['Fare', 'Deck']]<count_missing_values>
gc.collect()
pipeline = DataPipeline()
train, initial_ids = pipeline.fit_transform(train_path )
def handle_missing_data(data, miss_abs):
for col in data:
miss_prop = data[col].isna().sum() /len(data)
 if miss_prop < 0.5:
if data[col].dtype == "float64" and data[col].isnull().sum() > 0:
mean = data[col].mean()
data[col] = data[col].fillna(mean)
 print("Filling {} missing values in {} with mean value {:.0f}.".format(miss_abs[col], col, mean))
elif data[col].dtype == "object" and data[col].isnull().sum() > 0:
top = data[col].describe().top
data[col] = data[col].fillna(top)
 print("Filling {} missing values in {} with top value {}.".format(miss_abs[col], col, top))
return data<load_from_csv>
mean_vals = pd.Series()
std_vals = pd.Series()
for num, col in enumerate(train.columns):
if col not in ['winPlacePerc', 'groupId', 'matchId']:
train[col] = train[col].astype(np.float32)
 mean = train[col].mean()
stdev = train[col].std()
if stdev == 0:
train.drop(columns=col, inplace=True)
 else:
train[col] =(( train[col] - mean)/stdev ).astype(np.float32)
 mean_vals[col] = mean
std_vals[col] = stdev
gc.collect()
for key, dataset in data_dict.items() :
print('{} operations:'.format(key))
miss_abs = miss_data_dict[key]['{}: missing values(absolut)'.format(key)]
miss_rel = miss_data_dict[key]['{}: missing values(relative in %)'.format(key)]
data_dict[key] = handle_missing_data(dataset, miss_abs )<feature_engineering>
def get_data_batch(batch_size, randomize=False):
if randomize:
curr_idx = np.random.permutation(train.shape[0])
 else:
curr_idx = range(train.shape[0])
 for batch_n in range(int(np.ceil(train.shape[0] / batch_size))):
batch_data = train.iloc[curr_idx[batch_n*batch_size:(batch_n+1)*batch_size]]
group_sizes = batch_data['groupSize'].values*std_vals['groupSize'] + mean_vals['groupSize']
yield batch_data.drop(columns=['winPlacePerc', 'matchId', 'groupId'] ).values, batch_data['winPlacePerc'].values, group_sizes
def title_extract(data):
data['Title'] = data['Name'].str.split(',', expand=True)[1].str.split('.', expand=True)[0]<count_values>
dropout_rate = [0.0, 0.0, 0.0, 0.0, 0.0, 0.5, 0.5]
def leaky_relu(z, name=None):
return tf.maximum(0.01*z, z, name=name)
 tf.reset_default_graph()
dynamic_dropout = tf.placeholder_with_default(0.5, shape=(None), name='dynamic_dropout')
 X = tf.placeholder(dtype=tf.float32, shape=[None, train.shape[1]-3], name='X')
 y = tf.placeholder(dtype=tf.float32, shape=(None), name='y')
 group_sizes = tf.placeholder(dtype=tf.float32, shape=(None), name='group_sizes')
 training = tf.placeholder_with_default(False, shape=(None), name='training')
 with tf.name_scope('layers')as scope:
hidden1 = tf.layers.dense(X, 100, name='hidden1',
kernel_initializer=tf.variance_scaling_initializer())
 hidden1 = tf.layers.batch_normalization(hidden1, training=training, momentum=0.9, name='hidden1_bn')
 hidden1 = tf.nn.elu(hidden1, name='hidden1_activation')
 hidden2 = tf.layers.dense(hidden1, 60, name='hidden2',
kernel_initializer=tf.variance_scaling_initializer())
 hidden2 = tf.layers.batch_normalization(hidden2, training=training, momentum=0.9, name='hidden2_bn')
 hidden2 = tf.nn.elu(hidden2, name='hidden2_activation')
 hidden3 = tf.layers.dense(hidden2, 60, name='hidden3',
kernel_initializer=tf.variance_scaling_initializer())
 hidden3 = tf.layers.batch_normalization(hidden3, training=training, momentum=0.9, name='hidden3_bn')
 hidden3 = tf.nn.elu(hidden3, name='hidden3_activation')
 hidden4 = tf.layers.dense(hidden3, 60, name='hidden4',
kernel_initializer=tf.variance_scaling_initializer())
 hidden4 = tf.layers.batch_normalization(hidden4, training=training, momentum=0.9, name='hidden4_bn')
 hidden4 = tf.nn.elu(hidden4, name='hidden4_activation')
 hidden5 = tf.layers.dense(hidden4, 60, name='hidden5',
kernel_initializer=tf.variance_scaling_initializer())
 hidden5 = tf.layers.batch_normalization(hidden5, training=training, momentum=0.9, name='hidden5_bn')
 hidden5 = tf.nn.elu(hidden5, name='hidden5_activation')
 hidden5 = tf.layers.dropout(hidden5, rate=dropout_rate[5], training=training, name='dropout5')
 output = tf.layers.dense(hidden5, 1, name='output',
kernel_initializer=tf.variance_scaling_initializer() )
for key, dataset in data_dict.items() :
title = title_extract(dataset)
 data_all = concat_data(train_data, test_data)
 data_all['Title'].value_counts()<count_values>
with tf.name_scope('loss')as scope:
diff_vector = tf.abs(tf.reshape(output, shape=[-1])- y)
 sum_group_sizes = tf.reduce_sum(group_sizes)
 loss_mse = tf.reduce_sum(tf.square(diff_vector)*group_sizes)/sum_group_sizes
loss_mae = tf.reduce_sum(diff_vector*group_sizes)/sum_group_sizes
clipped_diff_vector = tf.clip_by_value(diff_vector, 0.0, 1.0)
 lin_err = 2*(diff_vector - clipped_diff_vector)
 loss_hybrid = tf.reduce_sum(( tf.square(clipped_diff_vector)+ lin_err)*group_sizes)/sum_group_sizes
with tf.name_scope('training')as scope:
lr_high = 0.0008
lr_low = 0.00001
learning_rate = tf.Variable(0.0001, trainable=False, name='learning_rate')
 optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate, beta1=0.6)
 training_op = optimizer.minimize(loss_hybrid)
 val_batch_size = 10000
def get_mse_mae(sess, batch_size=10000):
y_diffs = np.empty(shape=0)
 curr_group_sizes = np.empty(shape=0)
 for X_batch, y_batch, batch_gr_sizes in get_data_batch(batch_size=batch_size):
curr_output_vals = sess.run(output, feed_dict = {X:X_batch} ).reshape(-1)
 y_diffs = np.r_[y_diffs, np.abs(y_batch - curr_output_vals)]
curr_group_sizes = np.r_[curr_group_sizes, batch_gr_sizes]
loss_mae_val = np.sum(y_diffs*curr_group_sizes)/np.sum(curr_group_sizes)
 loss_mse_val = np.sum(( y_diffs**2)*curr_group_sizes)/np.sum(curr_group_sizes)
 return loss_mse_val, loss_mae_val
extra_training_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
 init = tf.global_variables_initializer()
saver = tf.train.Saver()
title_names =(data_all['Title'].value_counts() < 10)
 for key, dataset in data_dict.items() :
dataset['Title'] = dataset['Title'].apply(lambda x: 'Misc' if title_names.loc[x] == True else x)
 data_all = concat_data(train_data, test_data)
 print(data_all['Title'].value_counts() )<data_type_conversions>
gc.collect()
batch_size = 5000
n_epochs = 1000
max_epochs_wo_improvement = 50
max_epochs_wo_lr_change = 10
max_time = 2
loss_data = {'mse_train':[], 'mae_train':[], 'lr':[]}
start_time = datetime.datetime.now()
def get_dropout(initial_dropout, epoch):
return np.maximum(0, initial_dropout*(1 - epoch/200))
with tf.Session() as sess:
init.run()
min_loss_mae = np.inf
epochs_wo_improvement = 0
epochs_since_lr_change = 0
batch_lr = lr_high
for epoch in range(n_epochs):
for X_batch, y_batch, batch_gr_sizes in get_data_batch(batch_size=batch_size, randomize=True):
feed_dict = {X: X_batch, y: y_batch, training:True, learning_rate:batch_lr, group_sizes:batch_gr_sizes}
sess.run([training_op, extra_training_ops], feed_dict=feed_dict)
 loss_mse_train, loss_mae_train = get_mse_mae(sess)
 loss_data['mse_train'].append(loss_mse_train)
 loss_data['mae_train'].append(loss_mae_train)
 loss_data['lr'].append(batch_lr)
 print('epoch', epoch, 'time passed:',(datetime.datetime.now() - start_time), 'learning rate: {:<5.5f}'.format(batch_lr))
print('MSE:', loss_mse_train, 'MAE:', loss_mae_train)
 if loss_mae_train <= min_loss_mae:
epochs_wo_improvement = 0
epochs_since_lr_change = 0
min_loss_mae = loss_mae_train
save_path = saver.save(sess, '.. /DNN_data/dnn_state.ckpt')
 print('- best so far!')
 else:
epochs_wo_improvement += 1
epochs_since_lr_change += 1
if(epochs_wo_improvement > max_epochs_wo_improvement)or(( datetime.datetime.now() - start_time ).seconds > max_time*60*60):
print('early breaking!')
 break
if epochs_since_lr_change >= max_epochs_wo_lr_change:
batch_lr = batch_lr -(batch_lr - lr_low)*0.3
epochs_since_lr_change = 0
saver.restore(sess, save_path)
 loss_mse_val, loss_mae_val = get_mse_mae(sess)
 print('Final MSE:', loss_mse_val, 'final MAE:', loss_mae_val )
for key, dataset in data_dict.items() :
dataset['Age'] = dataset['Age'].astype('int', copy=True)
 age_labels = [1,2,3,4,5]
dataset['Age interval'] = pd.cut(data_all['Age'].astype('int'),5)
 dataset['Age Code'] = pd.cut(data_all['Age'],5, labels=age_labels ).astype('int64')
 data_all = concat_data(train_data, test_data)
 print('Number of cases within age intervals in datast:')
 data_all['Age interval'].value_counts()<categorify>
group_cols = [col for col in train.columns if re.search(r'group', col)is not None]
group_cols = [col for col in group_cols if re.search(r'_rank', col)is None]
group_cols.remove('groupId')
 other_cols = [col for col in train.columns if re.search(r'match|group', col)is None]
other_cols.remove('num_opponents')
 other_cols.remove('winPlacePerc' )
for key, dataset in data_dict.items() :
fare_labels = [1,2,3,4,5]
dataset['Fare interval'] = pd.qcut(data_all['Fare'], 5)
 dataset['Fare Code'] = pd.qcut(data_all['Fare'], 5, labels=fare_labels ).astype('int64')
 data_all = concat_data(train_data, test_data)
 print('Number of cases per fare category in dataset(approx.equal):')
 data_all['Fare interval'].value_counts()<concatenate>
def get_match_comp_indices(x, gaps):
gap_lim = x.shape[0]
gaps = [gap for gap in gaps if gap < gap_lim]
if len(gaps)> 0:
x = x.sort_values(by=['winPlacePerc'])
 results = []
for gap in gaps:
result = np.empty(shape=[x.shape[0]-gap, 2], dtype=np.int32)
 result[:,0] = np.array(x.iloc[gap:].index ).astype(np.int32)
 result[:,1] = np.array(x.iloc[:-gap].index ).astype(np.int32)
 results.append(result)
 return np.vstack(results)
 else:
return np.empty(shape=[0,2], dtype=np.int32)
 def get_comparator_indices(X, gap=1):
result = X[['matchId', 'winPlacePerc']].groupby(['matchId'] ).apply(lambda x: get_match_comp_indices(x, gap))
print('stage 1 complete')
 result = np.concatenate(result, axis=0)
 return result
for key, dataset in data_dict.items() :
dataset['Family Size'] = dataset ['SibSp'] + dataset['Parch'] + 1
data_all = concat_data(train_data, test_data )<count_values>
gaps = [i for i in range(1, 4)]
train_comp_inds = get_comparator_indices(train, gaps )
family_mapping =(lambda s: 1 if s == 1 else(2 if s == 2 else(3 if 3 <= s <= 4 else(4 if s >= 5 else 0))))
for key, dataset in data_dict.items() :
dataset['Family Size Code'] = dataset['Family Size'].map(family_mapping)
 data_all = concat_data(train_data, test_data)
 data_all['Family Size Code'].value_counts()<concatenate>
train_array = train.drop(columns=['matchId', 'groupId', 'winPlacePerc'] ).values
for key, dataset in data_dict.items() :
dataset['Mother'] = np.where(( dataset.Title == 'Mrs')&(dataset.Parch >0),1,0)
 data_all = concat_data(train_data, test_data )<categorify>
group_col_inds = [train.columns.drop(['matchId', 'groupId', 'winPlacePerc'] ).get_loc(col)for col in group_cols]
other_col_inds = [train.columns.drop(['matchId', 'groupId', 'winPlacePerc'] ).get_loc(col)for col in other_cols]
def get_comp_batch(inds, batch_size, omit_last=True):
rand_inds = np.random.permutation(inds.shape[0])
 if omit_last:
func = np.floor
else:
func = np.ceil
for batch_n in range(int(func(inds.shape[0]/batch_size))):
batch_inds = inds[rand_inds[batch_n*batch_size:(batch_n+1)*batch_size]]
X1_batch = train_array[batch_inds[:,0]][:,group_col_inds]
X2_batch = train_array[batch_inds[:,1]][:,group_col_inds]
X3_batch = train_array[batch_inds[:,0]][:,other_col_inds]
yield X1_batch, X2_batch, X3_batch
for key, dataset in data_dict.items() :
dataset['Ticket Frequency'] = data_all.groupby('Ticket')['Ticket'].transform('count')
 data_all = concat_data(train_data, test_data )<categorify>
dropout_rate = [0.0, 0.0, 0.0, 0.0, 0.1, 0.2, 0.2, 0.2]
layer_sizes = [100, 100, 60, 60, 60, 30, 30, 30]
tf.reset_default_graph()
def leaky_relu(z, name=None):
return tf.maximum(0.01*z, z, name=name)
 X1 = tf.placeholder(dtype=tf.float32, shape=[None, len(group_cols)], name='X1')
 X2 = tf.placeholder(dtype=tf.float32, shape=[None, len(group_cols)], name='X2')
 X3 = tf.placeholder(dtype=tf.float32, shape=[None, len(other_cols)], name='X3')
 X = tf.concat([tf.concat([X1, X2, X3], axis=1), tf.concat([X2, X1, X3], axis=1)], axis=0)
 y = tf.placeholder(dtype=tf.float32, shape=(None), name='y')
 y_all = tf.concat([y, 1-y], axis=0)
 training = tf.placeholder_with_default(False, shape=(None), name='training')
 with tf.name_scope('layers')as scope:
hidden = tf.layers.dropout(X, rate=dropout_rate[0], training=training, name='input_dropout')
 for i in range(8):
hidden = tf.layers.dense(hidden, layer_sizes[i], name=f'hidden_{i}',
kernel_initializer=tf.contrib.layers.variance_scaling_initializer())
 hidden = tf.layers.batch_normalization(hidden, training=training, momentum=0.9, name=f'hidden_{i}_bn')
 hidden = tf.nn.elu(hidden, name=f'hidden_{i}_activation')
 hidden = leaky_relu(hidden, name=f'hidden_{i}_activation')
 hidden = tf.layers.dropout(hidden, rate=dropout_rate[i], training=training, name=f'dropout_{i}')
 logits = tf.layers.dense(hidden, 1, name='logits',
kernel_initializer=tf.contrib.layers.variance_scaling_initializer())
 output = tf.nn.sigmoid(logits, name='output' )
X_train = pd.get_dummies(train_data[features])
 X_test = pd.get_dummies(test_data[ features])
 X_test.head()<prepare_x_and_y>
with tf.name_scope('loss')as scope:
xentropy = tf.nn.sigmoid_cross_entropy_with_logits(labels=y_all,logits=tf.reshape(logits, [-1]),name='xentropy')
 loss = tf.reduce_mean(xentropy)
 with tf.name_scope('training')as scope:
lr_low = 0.00001
lr_high = 0.003
lr_high_2 = 0.0003
decay_rate = 15
lr = tf.Variable(lr_high, trainable=False, name='learning_rate')
 optimizer = tf.train.AdamOptimizer(learning_rate=lr, epsilon=1e-10)
 training_op = optimizer.minimize(loss)
 with tf.name_scope('evaluation')as scope:
accuracy = tf.reduce_mean(tf.cast(tf.equal(tf.round(tf.reshape(output,[-1])) ,y_all), tf.float32), name='accuracy')
 def get_full_set_accuracy(sess, inds, batch_size=1000):
y_batch = np.ones(shape=[batch_size])
 y_for_comp = np.r_[y_batch, 1 - y_batch]
acc_data = np.empty(shape=0)
 for X1_batch, X2_batch, X3_batch in get_comp_batch(inds=inds, batch_size=batch_size):
feed_dict = {X1:X1_batch, X2:X2_batch, X3:X3_batch, y:y_batch}
y_pred = sess.run(output, feed_dict=feed_dict ).reshape(-1)
 batch_acc_data =(( y_pred > 0.5 ).astype(np.float32)== y_for_comp ).astype(np.float32)
 acc_data = np.r_[acc_data, batch_acc_data]
return np.mean(acc_data)
 extra_training_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
 init = tf.global_variables_initializer()
saver_comp = tf.train.Saver()
y = train_data['Survived']<choose_model_class>
gc.collect()
batch_size = 500
n_epochs = 1000
max_epochs_without_improving = 50
max_epochs_wo_lr_change = 7
max_time = 5.5
acc_data = {'acc_train':[], 'acc_val':[]}
batch_acc_data = []
batch_lr_data = []
start_time = datetime.datetime.now()
with tf.Session() as sess:
init.run()
max_val_acc = 0
epochs_wo_improvement = 0
epochs_since_lr_update = 0
gap = 1
batch_lr = lr_high
y_batch = np.ones(shape=[batch_size])
 for epoch in range(n_epochs):
n_batch = 0
if epoch == 15:
batch_size = 1000
y_batch = np.ones(shape=[batch_size])
 max_epochs_without_improving = 100
max_epochs_wo_lr_change = 10
if epoch == 25:
batch_size = 2000
y_batch = np.ones(shape=[batch_size])
 if epoch == 40:
batch_size = 4000
y_batch = np.ones(shape=[batch_size])
 max_epochs_wo_lr_change = 15
if epoch == 60:
batch_size = 8000
y_batch = np.ones(shape=[batch_size])
 max_epochs_wo_lr_change = 20
for X1_batch, X2_batch, X3_batch in get_comp_batch(inds=train_comp_inds, batch_size=batch_size):
feed_dict = {X1:X1_batch, X2:X2_batch, X3:X3_batch, y:y_batch}
batch_lr_data.append(batch_lr)
 batch_acc_data.append(sess.run(accuracy, feed_dict=feed_dict))
feed_dict = {X1:X1_batch, X2:X2_batch, X3:X3_batch, y:y_batch, training:True, lr:batch_lr}
sess.run([training_op, extra_training_ops], feed_dict=feed_dict)
 n_batch += 1
if(epoch >= 20)and(epoch % 1 == 0):
curr_ids = np.random.permutation(train_comp_inds.shape[0])[:100000]
train_acc = get_full_set_accuracy(sess, train_comp_inds[curr_ids], batch_size=10000)
 acc_data['acc_train'].append(train_acc)
 print('epoch', epoch, 'time passed:',(datetime.datetime.now() - start_time), 'max lr:', batch_lr)
 print('Accuracy: train:', train_acc)
 if train_acc > max_val_acc:
epochs_wo_improvement = 0
epochs_since_lr_update = 0
max_val_acc = train_acc
save_path_comp = saver_comp.save(sess, '.. /DNN_data/dnn_state_comp.ckpt')
 print('- best so far!')
 else:
epochs_wo_improvement += 1
epochs_since_lr_update += 1
if(epochs_wo_improvement > max_epochs_without_improving)or(( datetime.datetime.now() - kernel_start_time ).seconds/60/60 > max_time):
print('early breaking!')
 break
if epochs_since_lr_update >= max_epochs_wo_lr_change:
batch_lr = batch_lr -(batch_lr-lr_high_2)*0.33
epochs_since_lr_update = 0
saver_comp.restore(sess, save_path_comp)
 curr_ids = np.random.permutation(train_comp_inds.shape[0])[:100000]
train_acc = get_full_set_accuracy(sess, train_comp_inds[curr_ids], batch_size=10000)
 print('Final accuracy: train:', train_acc )
model = RandomForestClassifier(n_estimators=1800,
max_depth=8,
min_samples_split=6,
min_samples_leaf=6,
max_features='auto',
oob_score=True,
random_state=42,
n_jobs=-1,
verbose=1)
 model.fit(X_train, y)
 predictions = model.predict(X_test)
 acc_random_forest = round(model.score(X_train, y)* 100, 2)
 print(acc_random_forest)
 output = pd.DataFrame({'PassengerID': test_data.PassengerId, 'Survived': predictions})
 output = output.convert_dtypes()
output.to_csv('my_submission.csv', index=False)
 print("Your submission was successfully saved!")
 output.head()<set_options>
def generate_val_comp_data(X, gap):
if X.shape[0] > gap:
X = X.sort_values(by=['winPlacePerc'])
 group_data = X[group_cols].values
part0 = X[['matchId', 'groupId']].values[gap:,:]
part0 = np.c_[part0, X[['matchId', 'groupId']].values[:-gap,:]]
part1 = group_data[gap:,:]
part2 = group_data[:-gap,:]
part3 = X[other_cols].values[:-gap,:]
return part0, part1, part2, part3
return np.empty(shape=(0, 4)) , np.empty(shape=(0, len(group_cols))), np.empty(shape=(0, len(group_cols))), np.empty(shape=(0, len(other_cols)))
 def get_val_batch(data, batch_size=1000, gap=1):
matchIds = data['matchId'].unique().tolist()
for i in range(int(np.ceil(len(matchIds)/batch_size))):
match_ids_batch = matchIds[i*batch_size:(i+1)*batch_size]
grouped_data = data.loc[data['matchId'].isin(match_ids_batch)].groupby('matchId' ).apply(lambda x: generate_val_comp_data(x, gap))
part0, part1, part2, part3 = list(map(np.vstack, zip(*grouped_data)))
 yield part0, part1, part2, part3
%matplotlib inline
sns.set_style('whitegrid')
 ExtraTreesClassifier, GradientBoostingClassifier, VotingClassifier)
 LogisticRegression, PassiveAggressiveClassifier, RidgeClassifier)
 data_raw = pd.read_csv('/kaggle/input/titanic/train.csv')
 data_test = pd.read_csv('/kaggle/input/titanic/test.csv')
 train = data_raw.copy(deep=True)
 data_all = [train, data_test] 
<feature_engineering>
def reorder_val(data, ids, y_comp_pred):
df = pd.DataFrame(np.c_[ids, y_comp_pred.reshape(-1,1)], columns=['matchId', 'groupId', 'ordered'])
 data = data.merge(df, on=['matchId', 'groupId'], how='left')
 data['ordered'].fillna(1, inplace=True)
 data.sort_values(by=['matchId','winPlacePerc'], inplace=True)
 zero_inds = np.nonzero(data['ordered'].values == 0)[0]
i = 0
old_order = []
new_order = []
wPP_col_num = data.columns.get_loc('winPlacePerc')
 while i < len(zero_inds):
start = zero_inds[i]
while(i+1 < len(zero_inds)) and(zero_inds[i+1] - zero_inds[i] == 1):
i += 1
end = zero_inds[i]
length = end - start + 1
i += 1
old_order.extend([i for i in range(start-1, start+length, 1)])
 new_order.extend([i for i in range(start+length-1, start-2, -1)])
 data.iloc[old_order, wPP_col_num] = data.iloc[new_order, wPP_col_num].values
print('rows swapped')
 data.drop(columns=['ordered'], inplace=True)
 return data
for dataset in data_all:
dataset.drop(['Ticket', 'Cabin'], axis=1, inplace=True)
 dataset['Fare'].fillna(dataset['Fare'].median() , inplace=True)
 dataset['Embarked'].fillna(dataset['Embarked'].mode() [0], inplace=True)
 for c in set(dataset['Pclass']):
for s in set(dataset['Sex']):
age_median = dataset[(dataset['Pclass'] == c)&(dataset['Sex'] == s)]['Age'].median()
dataset.loc[(dataset['Age'].isnull())&(dataset['Pclass'] == c)&(dataset['Sex'] == s), 'Age'] = age_median
train.drop(['PassengerId'], axis=1, inplace=True)
 def get_titles(series):
return series.str.extract('([a-zA-Z]+)\.', expand=False)
 for dataset in data_all:
dataset['FamilySize'] = dataset['Parch'] + dataset['SibSp'] + 1
dataset['IsAlone'] =(dataset['FamilySize'] == 1 ).astype(int)
 dataset['Title'] = get_titles(dataset['Name'])
 title_counts = dataset['Title'].value_counts()
dataset['Title'] = dataset['Title'].map(lambda t: t if title_counts[t] >= 10 else 'Rare')
 dataset.drop('Name', axis=1, inplace=True)
 dataset['FareBin'] = pd.qcut(dataset['Fare'], 4)
 dataset['AgeBin'] = pd.cut(dataset['Age'], 5)
 categorical_cols = ['Sex', 'Embarked', 'Title', 'AgeBin', 'FareBin']
numerical_cols = ['Pclass', 'Age', 'SibSp', 'Parch', 'Fare', 'FamilySize', 'IsAlone']
le = LabelEncoder()
ohe = OneHotEncoder(sparse=False)
 for dataset in data_all:
for col in categorical_cols:
dataset[col + '_Code'] = le.fit_transform(dataset[col])
<drop_column>
train = None
train_idx = None
train_ids = None
gc.collect()
train = train.dropna(axis=0, subset=['Survived'])
 targ_col = 'Survived'
feature_cols = ['Pclass', 'FamilySize', 'IsAlone', 'Sex_Code', 'Embarked_Code',
'Title_Code', 'AgeBin_Code', 'FareBin_Code'] 
<choose_model_class>
test, test_ids = pipeline.transform(test_path)
 for num, col in enumerate(test.columns):
if col not in ['winPlacePerc', 'groupId', 'matchId']:
test[col] = test[col].astype(np.float32)
 test[col] =(( test[col] - mean_vals[col])/std_vals[col] ).astype(np.float32)
 gc.collect()
classifiers = [
KNeighborsClassifier(**{'n_jobs': -1, 'n_neighbors': 7}),
SVC(**{'C': 0.390625, 'kernel': 'poly', 'probability': True, 'random_state': 0}),
LinearSVC(**{'C': 25.0, 'dual': False, 'loss': 'squared_hinge', 'max_iter': 25000, 'penalty': 'l1', 'random_state': 0}),
DecisionTreeClassifier(**{'criterion': 'entropy', 'max_depth': 10, 'min_samples_split': 8, 'random_state': 0}),
RandomForestClassifier(**{'criterion': 'entropy', 'max_depth': 12, 'min_samples_split': 8, 'n_estimators': 300, 'n_jobs': -1, 'random_state': 0}),
AdaBoostClassifier(**{'algorithm': 'SAMME', 'learning_rate': 0.3, 'n_estimators': 300, 'random_state': 0}),
BaggingClassifier(**{'max_samples': 0.1, 'n_estimators': 50, 'n_jobs': -1, 'random_state': 0}),
ExtraTreesClassifier(**{'criterion': 'gini', 'max_depth': 12, 'min_samples_split': 12, 'n_estimators': 200, 'n_jobs': -1, 'random_state': 0}),
GradientBoostingClassifier(**{'learning_rate': 0.3, 'max_depth': 2, 'min_samples_split': 2, 'n_estimators': 10, 'random_state': 0}),
BernoulliNB(**{'alpha': 0.1}),
GaussianNB() ,
LinearDiscriminantAnalysis(**{'shrinkage': None, 'solver': 'svd'}),
QuadraticDiscriminantAnalysis() ,
Perceptron(**{'early_stopping': True, 'n_jobs': -1, 'penalty': 'l2', 'random_state': 0}),
LogisticRegression(**{'C': 0.09765625, 'max_iter': 20, 'n_jobs': -1, 'penalty': 'l2', 'random_state': 0, 'solver': 'lbfgs'}),
PassiveAggressiveClassifier(**{'C': 0.01220703125, 'early_stopping': True, 'loss': 'hinge', 'max_iter': 5, 'n_jobs': -1, 'random_state': 0}),
RidgeClassifier(**{'alpha': 0.5, 'normalize': True, 'random_state': 0}),
GaussianProcessClassifier(**{'max_iter_predict': 10, 'n_jobs': -1, 'random_state': 0}),
XGBClassifier(**{'booster': 'gbtree', 'learning_rate': 0.1, 'max_depth': 10, 'n_jobs': -1, 'random_state': 0, 'reg_alpha': 0.16, 'reg_lambda': 2.56}),
LGBMClassifier(**{'boosting_type': 'goss', 'learning_rate': 0.1, 'n_estimators': 1000, 'n_jobs': -1, 'random_state': 0, 'reg_alpha': 0.16, 'reg_lambda': 0}),
]
skf = StratifiedKFold(n_splits=10, shuffle=True, random_state=0 )<train_model>
y_test_dnn = pd.DataFrame(np.zeros(shape=[1,3]), columns=['matchId', 'groupId', 'winPlacePerc_pred'])
 predict_batch_size = 10000
tf.reset_default_graph()
saver = tf.train.import_meta_graph('.. /DNN_data/dnn_state.ckpt.meta')
 X = tf.get_default_graph().get_tensor_by_name('X:0')
 output = tf.get_default_graph().get_tensor_by_name('layers/output/BiasAdd:0')
 init = tf.global_variables_initializer()
with tf.Session() as sess:
init.run()
saver.restore(sess, '.. /DNN_data/dnn_state.ckpt')
 n_batches = int(np.ceil(test.shape[0]/predict_batch_size))
for batch_n in range(n_batches):
data_batch = test.iloc[batch_n*predict_batch_size:(batch_n+1)*predict_batch_size]
X_batch = data_batch.drop(columns=['matchId','groupId'])
 y_batch = data_batch[['matchId','groupId']]
y_batch['winPlacePerc_pred'] = output.eval(session=sess, feed_dict={X: X_batch})
 y_test_dnn = y_test_dnn.append(y_batch, ignore_index=True)
 y_test_dnn = y_test_dnn.iloc[1:,:]
clf_scores = pd.DataFrame(columns=['Classifier', 'Test Score', 'Test Score 3*STD'])
 clf_preds = pd.DataFrame(train[targ_col])
 for i in range(len(classifiers)) :
clf = classifiers[i]
clf_name = clf.__class__.__name__
clf.fit(train[feature_cols], train[targ_col])
 cv_results = cross_val_score(clf, train[feature_cols], train[targ_col], cv=skf)
 clf_scores.loc[i] = [clf_name, cv_results.mean() , cv_results.std() * 3]
clf_preds[clf_name] = clf.predict(train[feature_cols])
 clf_scores<compute_test_metric>
def rank_align_predictions(X, y, scaled=False):
X['winPlacePerc'] = y
X['rank'] = X.groupby(['matchId'])['winPlacePerc'].rank(method='dense')
 X['max_rank'] = X.groupby(['matchId'])['rank'].transform(np.max)
 adj_winPlacePerc =(X['rank'] - 1)/(X['max_rank'] - 1 + 0.0000000001)
 X.drop(columns=['winPlacePerc', 'rank', 'max_rank'], inplace=True)
 return adj_winPlacePerc
def fix_predictions(X, y, scaled=False):
y = y.copy()
y[y > 1.0] = 1.0
y[y < 0.0] = 0.0
if scaled:
max_places = X['maxPlace'].values*std_vals['maxPlace'] + mean_vals['maxPlace']
num_groups = X['numGroups'].values*std_vals['numGroups'] + mean_vals['numGroups']
else:
max_places = X['maxPlace'].values
num_groups = X['numGroups'].values
multiplier =(max_places[max_places > 1] - 1 ).astype(np.float32)
 y[max_places > 1] = np.round(y[max_places > 1] * multiplier)/ multiplier
y[max_places == 1] = 1.0
y[max_places <= 0] = 0.0
y[num_groups <= 1] = 0.0
return y
def print_scores_info(model_name, scores):
mean = scores.mean() * 100
std_3 = scores.std() * 100 * 3
print(model_name, 'score mean: ', mean)
 print(model_name, 'score 3 std range: ', mean - std_3, '—', mean + std_3 )<choose_model_class>
y_test_dnn['winPlacePerc_pred'] = rank_align_predictions(y_test_dnn.drop(columns=['winPlacePerc_pred']), y_test_dnn['winPlacePerc_pred'])
 y_test_dnn = y_test_dnn.merge(test[['matchId', 'groupId', 'numGroups', 'maxPlace']], on=['matchId', 'groupId'])
 y_test_dnn['winPlacePerc_pred'] = fix_predictions(y_test_dnn.drop(columns=['winPlacePerc_pred']), y_test_dnn['winPlacePerc_pred'], scaled=True )
vote_classifiers = [ 
('knn', KNeighborsClassifier(**{'n_jobs': -1, 'n_neighbors': 7})) , 
('bag', BaggingClassifier(**{'max_samples': 0.1, 'n_estimators': 50, 'n_jobs': -1, 'random_state': 0})) , 
('gbc', GradientBoostingClassifier(**{'learning_rate': 0.3, 'max_depth': 2, 'min_samples_split': 2, 'n_estimators': 10, 'random_state': 0})) , 
('bnb', BernoulliNB(**{'alpha': 0.1})) , 
('gnb', GaussianNB()), 
('lda', LinearDiscriminantAnalysis(**{'shrinkage': None, 'solver': 'svd'})) , 
('qda', QuadraticDiscriminantAnalysis()), 
('log', LogisticRegression(**{'C': 0.09765625, 'max_iter': 20, 'n_jobs': -1, 'penalty': 'l2', 'random_state': 0, 'solver': 'lbfgs'})) , 
('gpc', GaussianProcessClassifier(**{'max_iter_predict': 10, 'n_jobs': -1, 'random_state': 0})) , 
('xgb', XGBClassifier(**{'booster': 'gbtree', 'learning_rate': 0.1, 'max_depth': 10, 'n_jobs': -1, 'random_state': 0, 'reg_alpha': 0.16, 'reg_lambda': 2.56})) ,
]
vote_hard_clf = VotingClassifier(vote_classifiers, voting='hard')
 print_scores_info('Vote Hard', cross_val_score(vote_hard_clf, train[feature_cols], train[targ_col], cv=skf))
vote_hard_clf.fit(train[feature_cols], train[targ_col])
 preds_test = vote_hard_clf.predict(data_test[feature_cols])
 output = pd.DataFrame({'PassengerId': data_test.PassengerId, 'Survived': preds_test})
 output.to_csv('submission.csv', index=False )<set_options>
test = test.merge(y_test_dnn[['matchId', 'groupId', 'winPlacePerc_pred']], on=['matchId', 'groupId'])
 test.rename(columns={'winPlacePerc_pred':'winPlacePerc'}, inplace=True )
pd.options.mode.chained_assignment = None
warnings.filterwarnings('ignore' )<load_from_csv>
n_epochs = 6
batch_size = 10000
comp_thresh = [-0.1, 0, -0.1]
y_comp_diff_data = {}
y_comp_pred_data = {}
tf.reset_default_graph()
saver_comp = tf.train.import_meta_graph('.. /DNN_data/dnn_state_comp.ckpt.meta')
 X1 = tf.get_default_graph().get_tensor_by_name('X1:0')
 X2 = tf.get_default_graph().get_tensor_by_name('X2:0')
 X3 = tf.get_default_graph().get_tensor_by_name('X3:0')
 output = tf.get_default_graph().get_tensor_by_name('layers/output:0')
 init = tf.global_variables_initializer()
with tf.Session() as sess:
saver_comp.restore(sess, save_path_comp)
 for epoch in range(n_epochs):
y_comp_pred = np.empty(shape=[0])
 y_comp_diff = np.empty(shape=[0])
 all_ids = np.empty(shape=[0, 2])
 for batch_ids, X1_batch, X2_batch, X3_batch in get_val_batch(test, batch_size):
feed_dict = {X1:X1_batch, X2:X2_batch, X3:X3_batch}
batch_output = output.eval(feed_dict=feed_dict ).reshape(-1)
 assert len(batch_output)% 2 ==0
y_batch_pred = batch_output[:int(len(batch_output)/2)]
y_batch_comp = batch_output[:int(len(batch_output)/2)] - batch_output[int(len(batch_output)/2):]
y_comp_pred = np.r_[y_comp_pred, y_batch_pred]
y_comp_diff = np.r_[y_comp_diff, y_batch_comp]
all_ids = np.r_[all_ids, batch_ids[:,:2]]
y_comp_pred_data[epoch] = y_comp_pred
y_comp_diff_data[epoch] = y_comp_diff
thresh_ind = np.minimum(len(comp_thresh)-1, epoch)
 y_comp_diff = y_comp_diff > comp_thresh[thresh_ind]
print('epoch:', epoch, 'number of zeros:', len(np.nonzero(y_comp_diff == 0)[0]))
if np.min(y_comp_diff)== 0:
test = reorder_val(test, all_ids, y_comp_diff)
 else:
break
train = pd.read_csv('.. /input/train.csv', header=0)
 test = pd.read_csv('.. /input/test.csv', header=0)
 test.insert(1,'Survived',np.nan)
 all = pd.concat([train, test] )<feature_engineering>
def create_submission_table(X, y, id_table):
out = X[['matchId', 'groupId']]
out['winPlacePerc'] = y
out = id_table.merge(out, on=['matchId', 'groupId'])
 out = out.drop(columns=['groupId', 'matchId'])
 return out
<feature_engineering><EOS> 
submission = create_submission_table(test, test['winPlacePerc'], test_ids)
 submission.to_csv('submission.csv', index=False)
 submission.head(50 )