Kaggle competition: Santander Customer Transaction Prediction
<SOS> metric: AUC Kaggle data source: santander-customer-transaction-prediction<save_to_csv>
warnings.filterwarnings('ignore' )
df_oof = pd.DataFrame()
df_oof['lgb_oof'] = oof
df_oof.to_csv('light_gbm_oof.csv', index_label='id' )<save_to_csv>
%%time
train_df = pd.read_csv(PATH+"train.csv")
 test_df = pd.read_csv(PATH+"test.csv" )
df_submission = pd.DataFrame(index=df_test_id.index)
 df_submission['winPlacePerc'] = y
df_submission.to_csv('light_gbm_raw.csv', index_label='Id' )<load_from_csv>
def missing_data(data):
total = data.isnull().sum()
percent =(data.isnull().sum() /data.isnull().count() *100)
 tt = pd.concat([total,percent],axis=1,keys=['Total','Percent'])
 types = []
for col in data.columns:
dtype = str(data[col].dtype)
 types.append(dtype)
 tt['Types'] = types
return(np.transpose(tt))
df_test = pd.read_csv('.. /input/test_V2.csv')
 df_test.shape<merge>
%%time
missing_data(train_df )
df_submission = df_submission.merge(df_test[['Id', 'matchId', 'groupId', 'maxPlace', 'numGroups']], on='Id', how='left')
 df_submission.head()<merge>
%%time
missing_data(test_df )
df_submission_group = df_submission.groupby(['matchId', 'groupId'] ).first().reset_index()
df_submission_group['rank'] = df_submission_group.groupby(['matchId'])['winPlacePerc'].rank()
df_submission_group = df_submission_group.merge(df_submission_group.groupby('matchId')['rank'].max().to_frame('max_rank' ).reset_index() , on='matchId', how='left')
 df_submission_group['adjusted_perc'] =(df_submission_group['rank'] - 1)/(df_submission_group['numGroups'] - 1)
 df_submission = df_submission.merge(df_submission_group[['adjusted_perc', 'matchId', 'groupId']], on=['matchId', 'groupId'], how='left')
 df_submission['winPlacePerc'] = df_submission['adjusted_perc']
df_submission.head()<feature_engineering>
print("There are {}% target values with 1".format(100 * train_df["target"].value_counts() [1]/train_df.shape[0]))
df_submission.loc[df_submission.maxPlace == 0, 'winPlacePerc'] = 0
df_submission.loc[df_submission.maxPlace == 1, 'winPlacePerc'] = 1<feature_engineering>
correlations = train_df[features].corr().where(np.triu(np.ones(train_df[features].corr().shape),k=1 ).astype(np.bool))
correlations_df = correlations.abs().unstack().dropna().sort_values().reset_index()
correlations_df.shape
t = df_submission.loc[df_submission.maxPlace > 1]
gap = 1.0 /(t.maxPlace.values - 1)
 fixed_perc = np.around(t.winPlacePerc.values / gap)* gap
df_submission.loc[df_submission.maxPlace > 1, 'winPlacePerc'] = fixed_perc<feature_engineering>
%%time
features = train_df.columns.values[2:202]
unique_max_train = []
unique_max_test = []
for feature in features:
values = train_df[feature].value_counts()
unique_max_train.append([feature,values.max() ,values.idxmax() ])
 values = test_df[feature].value_counts()
unique_max_test.append([feature,values.max() ,values.idxmax() ] )
df_submission.loc[(df_submission.maxPlace > 1)&(df_submission.numGroups == 1), 'winPlacePerc'] = 0
assert df_submission['winPlacePerc'].isnull().sum() == 0<save_to_csv>
%%time
i = 1
for df in [test_df, train_df]:
idx = df.columns.values[i:i+200]
df['sum'] = df[idx].sum(axis=1)
 df['min'] = df[idx].min(axis=1)
 df['max'] = df[idx].max(axis=1)
 df['mean'] = df[idx].mean(axis=1)
 df['std'] = df[idx].std(axis=1)
 df['skew'] = df[idx].skew(axis=1)
 df['kurt'] = df[idx].kurtosis(axis=1)
 df['med'] = df[idx].median(axis=1)
 df['range'] = df['max']-df['min']
i = i + 1
df_submission[['Id', 'winPlacePerc']].to_csv('light_gbm_adjusted.csv', index=False )<set_options>
drop_cols = [col for col in correlations.columns if any(abs(correlations[col])>0.95)]
warnings.filterwarnings('ignore')
 INPUT_DIR = ".. /input/"<feature_engineering>
train_df = train_df.drop(columns=drop_cols)
 test_df = test_df.drop(columns=drop_cols )
def dataproc1(df):
df['playersJoined'] = df.groupby('matchId')['matchId'].transform('count')
 df['headshotrate'] = df['kills']/df['headshotKills']
df['killStreakrate'] = df['killStreaks']/df['kills']
df['DBNOs_over_kills'] = df['DBNOs']/df['kills']
df['healthitems'] = df['heals'] + df['boosts']
df['heals_over_boosts'] = df['heals'] / df['boosts']
df['healthitems_norm'] = df['heals']/1.37 + df['boosts']/1.1
df['healthitems_over_kills'] = df['healthitems'] / df['kills']
df['healthitems_norm_over_kills'] = df['healthitems_norm'] / df['kills']
df['walkDistance_over_heals'] = df['walkDistance'] / df['heals']
df['walkDistance_x_heals'] = df['walkDistance'] * df['heals']
df['walkDistance_over_kills'] = df['walkDistance'] / df['kills']
df['walkDistance_x_kills'] = df['walkDistance'] * df['kills']
df['walkDistance_over_healthitems'] = df['walkDistance'] / df['healthitems']
df['walkDistance_x_healthitems'] = df['walkDistance'] * df['healthitems']
df["skill"] = df["headshotKills"] + df["roadKills"]
df['killPlace_over_maxPlace'] = df['killPlace'] / df['maxPlace']
df['killPlace_over_numGroups'] = df['killPlace'] / df['numGroups']
df['rideDistance'] =(df['rideDistance']/500)
 df['walkDistance'] =(df['walkDistance']/500)
 df['swimDistance'] =(df['swimDistance']/500)
 df["total_time_by_distance"] =(df["rideDistance"]/4.5+df["walkDistance"]+df["swimDistance"]*1.5)
 df['totalDistance'] = df['rideDistance'] + df["walkDistance"] + df["swimDistance"]
df['distance_over_weapons'] = df['totalDistance'] / df['weaponsAcquired']
df['distance_x_weapons'] = df['totalDistance'] * df['weaponsAcquired']
df['total_time_by_distance_over_weapons'] = df['total_time_by_distance'] / df['weaponsAcquired']
df['total_time_by_distance_x_weapons'] = df['total_time_by_distance'] * df['weaponsAcquired']
df['killPlace_over_total_time_by_distance'] = df['killPlace'] / df['total_time_by_distance']
df['killPlace_x_total_time_by_distance'] = df['killPlace'] * df['total_time_by_distance']
df['killPlace_over_totalDistance'] = df['killPlace'] / df['totalDistance']
df['killPlace_x_totalDistance'] = df['killPlace'] * df['totalDistance']
df['boosts_over_total_time_by_distance'] = df['boosts'] / df['total_time_by_distance']
df['boosts_x_total_time_by_distance'] = df['boosts'] * df['total_time_by_distance']
df['boosts_over_totalDistance'] = df['boosts'] / df['totalDistance']
df['boosts_x_totalDistance'] = df['boosts'] * df['totalDistance']
df['teamwork'] = df['assists'] + df['revives'] - df['teamKills']
df['total_items_acquired'] =(df["boosts"] + df["heals"] + df["weaponsAcquired"])
 df['total_items_acquired_norm'] =(df["boosts"]/1.1 + df["heals"]/1.37 + df["weaponsAcquired"]/3.66)
 df['total_items_acquired_over_total_time_by_distance'] = df['total_items_acquired'] / df['total_time_by_distance']
df['total_items_acquired_x_total_time_by_distance'] = df['total_items_acquired'] * df['total_time_by_distance']
df['total_items_acquired_norm_over_total_time_by_distance'] = df['total_items_acquired_norm'] / df['total_time_by_distance']
df['total_items_acquired_norm_x_total_time_by_distance'] = df['total_items_acquired_norm'] * df['total_time_by_distance']
df['heals_over_total_time_by_distance'] = df['heals'] / df['total_time_by_distance']
df['heals_x_total_time_by_distance'] = df['heals'] * df['total_time_by_distance']
df['heals_over_totalDistance'] = df['heals'] / df['totalDistance']
df['heals_x_totalDistance'] = df['heals'] * df['totalDistance']
df['kills_over_total_time_by_distance'] = df['kills'] / df['total_time_by_distance']
df['kills_x_total_time_by_distance'] = df['kills'] * df['total_time_by_distance']
df['kills_over_totalDistance'] = df['kills'] / df['totalDistance']
df['kills_x_totalDistance'] = df['kills'] * df['totalDistance']
df['killsNorm'] = df['kills']*(( 100-df['playersJoined'])/100 + 1)
 df['damageDealtNorm'] = df['damageDealt']*(( 100-df['playersJoined'])/100 + 1)
 df['maxPlaceNorm'] = df['maxPlace']*(( 100-df['playersJoined'])/100 + 1)
 df['killPlace_over_maxPlaceNorm'] = df['killPlace'] / df['maxPlaceNorm']
df['killPlace_over_playersJoined'] = df['killPlace'] / df['playersJoined']
df['matchDurationNorm'] = df['matchDuration']*(( 100-df['playersJoined'])/100 + 1)
 df['killPlace_over_matchDuration'] = df['killPlace'] / df['matchDuration']
df['killPlace_over_matchDurationnorm'] = df['killPlace'] / df['matchDurationNorm']
df['killPlacePerc'] =(df['playersJoined'] - df['killPlace'])/(df['playersJoined'] - 1)
 df['L1'] = df['roadKills'] + df['vehicleDestroys'] + df['teamKills']
df['L2'] = df['revives'] + df['headshotKills'] + df['assists']
df['L3'] = df['killStreaks'] + df['DBNOs'] + df['kills'] + df['boosts'] + df['heals']
df['points'] = df['killPoints']+df['rankPoints'] + df['winPoints']
df['L1_over_total_time_by_distance'] = df['L1'] / df['total_time_by_distance']
df['L1_x_total_time_by_distance'] = df['L1'] * df['total_time_by_distance']
df['L1_over_totalDistance'] = df['L1'] / df['totalDistance']
df['L1_x_totalDistance'] = df['L1'] * df['totalDistance']
df['L2_over_total_time_by_distance'] = df['L2'] / df['total_time_by_distance']
df['L2_x_total_time_by_distance'] = df['L2'] * df['total_time_by_distance']
df['L2_over_totalDistance'] = df['L2'] / df['totalDistance']
df['L2_x_totalDistance'] = df['L2'] * df['totalDistance']
df['L3_over_total_time_by_distance'] = df['L3'] / df['total_time_by_distance']
df['L3_x_total_time_by_distance'] = df['L3'] * df['total_time_by_distance']
df['L3_over_totalDistance'] = df['L3'] / df['totalDistance']
df['L3_x_totalDistance'] = df['L3'] * df['totalDistance']
df[df == np.Inf] = np.NaN
df[df == np.NINF] = np.NaN
df.fillna(0, inplace=True)
 reduce_mem_usage(df)
 return df
def dataproc2(df, is_train='TRUE'):
print("Starting dataproc2")
 features = list(df.columns)
 if is_train=='TRUE':
features.remove(target)
 features.remove("Id")
 features.remove("matchId")
 features.remove("groupId")
 features.remove("matchType")
 features.remove("numGroups")
 features.remove("playersJoined")
 features.remove("roadKills")
 features.remove("vehicleDestroys")
 imp_cols=['matchId', 'groupId']
print("Step 1")
 tmp = df.groupby(['matchId','groupId'])[features].agg('mean')
 df_out = tmp.reset_index() [['matchId','groupId']]
if is_train=='TRUE':
imp_cols.extend([target])
 tmp3 = df.groupby(['matchId','groupId'])[target].agg('mean')
 df_out = df_out.merge(tmp3.reset_index() , how='left', on=['matchId','groupId'])
 del tmp3
gc.collect()
print("Step 2")
 features.remove("maxPlace")
 features.remove("matchDuration")
 tmp2 = tmp.groupby('matchId')[features].rank(pct=True)
 df_out = df_out.merge(tmp.add_suffix('_mean' ).reset_index() , how='left', on=['matchId', 'groupId'])
 imp_cols.extend(['killPlace_mean', 'matchDuration_mean', 'killPlace_over_maxPlace_mean', 'killPlace_over_numGroups_mean', 'boosts_x_total_time_by_distance_mean', 'killsNorm_mean', 'killPlace_over_playersJoined_mean', 'killPlacePerc_mean'])
 df_out = df_out[imp_cols]
df_out = df_out.merge(tmp2.add_suffix('_mean_rank' ).reset_index() , how='left', on=['matchId', 'groupId'])
 imp_cols.extend(['DBNOs_mean_rank', 'kills_mean_rank', 'killStreaks_mean_rank', 'longestKill_mean_rank', 'walkDistance_mean_rank', 'killStreakrate_mean_rank', 'walkDistance_x_kills_mean_rank', 'total_time_by_distance_mean_rank', 'total_time_by_distance_x_weapons_mean_rank', 'killPlace_over_total_time_by_distance_mean_rank', 'killPlace_x_total_time_by_distance_mean_rank', 'boosts_x_total_time_by_distance_mean_rank', 'total_items_acquired_x_total_time_by_distance_mean_rank', 'total_items_acquired_norm_x_total_time_by_distance_mean_rank', 'kills_over_total_time_by_distance_mean_rank', 'kills_x_total_time_by_distance_mean_rank', 'kills_x_totalDistance_mean_rank', 'killsNorm_mean_rank', 'killPlacePerc_mean_rank'])
 df_out = df_out[imp_cols]
del tmp
del tmp2
gc.collect()
print("Step 3")
 tmp = df.groupby(['matchId','groupId'])[features].agg('median')
 tmp2 = tmp.groupby('matchId')[features].rank(pct=True)
 df_out = df_out.merge(tmp.add_suffix('_median' ).reset_index() , how='left', on=['matchId', 'groupId'])
 imp_cols.extend(['killPlace_over_numGroups_median', 'killsNorm_median', 'killPlace_over_playersJoined_median', 'killPlacePerc_median'])
 df_out = df_out[imp_cols]
df_out = df_out.merge(tmp2.add_suffix('_median_rank' ).reset_index() , how='left', on=['matchId', 'groupId'])
 imp_cols.extend(['assists_median_rank', 'killPlace_median_rank', 'kills_median_rank', 'killStreaks_median_rank', 'longestKill_median_rank', 'revives_median_rank', 'walkDistance_median_rank', 'killStreakrate_median_rank', 'DBNOs_over_kills_median_rank', 'walkDistance_over_kills_median_rank', 'walkDistance_x_kills_median_rank', 'killPlace_over_numGroups_median_rank', 'total_time_by_distance_median_rank', 'total_time_by_distance_x_weapons_median_rank', 'killPlace_over_total_time_by_distance_median_rank', 'killPlace_x_total_time_by_distance_median_rank', 'killPlace_over_totalDistance_median_rank', 'total_items_acquired_x_total_time_by_distance_median_rank', 'total_items_acquired_norm_x_total_time_by_distance_median_rank', 'kills_over_total_time_by_distance_median_rank', 'kills_x_total_time_by_distance_median_rank', 'kills_over_totalDistance_median_rank', 'kills_x_totalDistance_median_rank', 'killsNorm_median_rank', 'killPlace_over_matchDuration_median_rank', 'killPlacePerc_median_rank', 'L3_over_total_time_by_distance_median_rank'])
 df_out = df_out[imp_cols]
del tmp
del tmp2
gc.collect()
print("Step 4")
 tmp = df.groupby(['matchId','groupId'])[features].agg('max')
 tmp2 = tmp.groupby('matchId')[features].rank(pct=True)
 df_out = df_out.merge(tmp.add_suffix('_max' ).reset_index() , how='left', on=['matchId', 'groupId'])
 imp_cols.extend(['damageDealt_max', 'killPlace_max', 'kills_max', 'walkDistance_max', 'walkDistance_over_kills_max', 'killPlace_over_maxPlace_max', 'killPlace_over_numGroups_max', 'boosts_x_total_time_by_distance_max', 'killsNorm_max', 'killPlace_over_maxPlaceNorm_max', 'killPlace_over_playersJoined_max', 'killPlacePerc_max'])
 df_out = df_out[imp_cols]
df_out = df_out.merge(tmp2.add_suffix('_max_rank' ).reset_index() , how='left', on=['matchId', 'groupId'])
 imp_cols.extend(['boosts_max_rank', 'DBNOs_max_rank', 'killPlace_max_rank', 'kills_max_rank', 'killStreaks_max_rank', 'longestKill_max_rank', 'walkDistance_max_rank', 'weaponsAcquired_max_rank', 'killStreakrate_max_rank', 'walkDistance_over_kills_max_rank', 'killPlace_over_maxPlace_max_rank', 'killPlace_over_numGroups_max_rank', 'total_time_by_distance_max_rank', 'totalDistance_max_rank', 'distance_x_weapons_max_rank', 'total_time_by_distance_x_weapons_max_rank', 'killPlace_over_total_time_by_distance_max_rank', 'killPlace_x_total_time_by_distance_max_rank', 'boosts_x_total_time_by_distance_max_rank', 'boosts_x_totalDistance_max_rank', 'total_items_acquired_x_total_time_by_distance_max_rank', 'total_items_acquired_norm_x_total_time_by_distance_max_rank', 'kills_x_total_time_by_distance_max_rank', 'kills_x_totalDistance_max_rank', 'killsNorm_max_rank', 'killPlace_over_maxPlaceNorm_max_rank', 'killPlace_over_playersJoined_max_rank', 'killPlace_over_matchDuration_max_rank', 'killPlace_over_matchDurationnorm_max_rank', 'killPlacePerc_max_rank', 'points_max_rank', 'L3_x_totalDistance_max_rank'])
 df_out = df_out[imp_cols]
del tmp
del tmp2
gc.collect()
print("Step 5")
 tmp = df.groupby(['matchId','groupId'])[features].agg('min')
 tmp2 = tmp.groupby('matchId')[features].rank(pct=True)
 df_out = df_out.merge(tmp.add_suffix('_min' ).reset_index() , how='left', on=['matchId', 'groupId'])
 imp_cols.extend(['kills_min', 'longestKill_min', 'killStreakrate_min', 'walkDistance_over_kills_min', 'killPlace_over_maxPlace_min', 'killPlace_over_numGroups_min', 'killPlace_over_total_time_by_distance_min', 'kills_over_total_time_by_distance_min', 'killsNorm_min', 'killPlace_over_maxPlaceNorm_min', 'killPlace_over_playersJoined_min', 'killPlace_over_matchDuration_min', 'killPlace_over_matchDurationnorm_min', 'killPlacePerc_min'])
 df_out = df_out[imp_cols]
df_out = df_out.merge(tmp2.add_suffix('_min_rank' ).reset_index() , how='left', on=['matchId', 'groupId'])
 imp_cols.extend(['assists_min_rank', 'DBNOs_min_rank', 'kills_min_rank', 'killStreaks_min_rank', 'longestKill_min_rank', 'rankPoints_min_rank', 'revives_min_rank', 'swimDistance_min_rank', 'walkDistance_min_rank', 'weaponsAcquired_min_rank', 'killStreakrate_min_rank', 'DBNOs_over_kills_min_rank', 'heals_over_boosts_min_rank', 'healthitems_over_kills_min_rank', 'healthitems_norm_over_kills_min_rank', 'walkDistance_over_kills_min_rank', 'walkDistance_x_kills_min_rank', 'killPlace_over_maxPlace_min_rank', 'total_time_by_distance_min_rank', 'total_time_by_distance_x_weapons_min_rank', 'killPlace_over_total_time_by_distance_min_rank', 'killPlace_x_total_time_by_distance_min_rank', 'killPlace_over_totalDistance_min_rank', 'teamwork_min_rank', 'total_items_acquired_x_total_time_by_distance_min_rank', 'kills_over_total_time_by_distance_min_rank', 'kills_x_total_time_by_distance_min_rank', 'kills_over_totalDistance_min_rank', 'kills_x_totalDistance_min_rank', 'killsNorm_min_rank', 'damageDealtNorm_min_rank', 'killPlace_over_maxPlaceNorm_min_rank', 'killPlace_over_matchDuration_min_rank', 'killPlacePerc_min_rank', 'L3_min_rank', 'points_min_rank', 'L3_over_total_time_by_distance_min_rank'])
 df_out = df_out[imp_cols]
del tmp
del tmp2
gc.collect()
print("Step 6")
 tmp = df.groupby(['matchId'])[features].agg('mean')
 df_out = df_out.merge(tmp.add_suffix('_match_mean' ).reset_index() , how='left', on=['matchId'])
 imp_cols.extend(['assists_match_mean', 'boosts_match_mean', 'damageDealt_match_mean', 'headshotKills_match_mean', 'heals_match_mean', 'kills_match_mean', 'killStreaks_match_mean', 'longestKill_match_mean', 'swimDistance_match_mean', 'weaponsAcquired_match_mean', 'headshotrate_match_mean', 'killStreakrate_match_mean', 'DBNOs_over_kills_match_mean', 'heals_over_boosts_match_mean', 'healthitems_over_kills_match_mean', 'healthitems_norm_over_kills_match_mean', 'walkDistance_over_heals_match_mean', 'walkDistance_over_kills_match_mean', 'walkDistance_x_kills_match_mean', 'walkDistance_over_healthitems_match_mean', 'skill_match_mean', 'distance_over_weapons_match_mean', 'total_time_by_distance_over_weapons_match_mean', 'killPlace_over_total_time_by_distance_match_mean', 'killPlace_x_total_time_by_distance_match_mean', 'killPlace_over_totalDistance_match_mean', 'killPlace_x_totalDistance_match_mean', 'boosts_over_total_time_by_distance_match_mean', 'boosts_over_totalDistance_match_mean', 'teamwork_match_mean', 'total_items_acquired_norm_match_mean', 'total_items_acquired_over_total_time_by_distance_match_mean', 'total_items_acquired_norm_over_total_time_by_distance_match_mean', 'heals_over_total_time_by_distance_match_mean', 'heals_over_totalDistance_match_mean', 'kills_over_total_time_by_distance_match_mean', 'kills_x_total_time_by_distance_match_mean', 'kills_over_totalDistance_match_mean', 'kills_x_totalDistance_match_mean', 'killsNorm_match_mean', 'damageDealtNorm_match_mean', 'L2_match_mean', 'points_match_mean', 'L1_over_totalDistance_match_mean', 'L2_over_total_time_by_distance_match_mean', 'L2_x_total_time_by_distance_match_mean', 'L2_over_totalDistance_match_mean', 'L2_x_totalDistance_match_mean', 'L3_over_total_time_by_distance_match_mean', 'L3_over_totalDistance_match_mean'])
 df_out = df_out[imp_cols]
del tmp
gc.collect()
tmp = df.groupby(['matchId'])[features].agg('median')
 df_out = df_out.merge(tmp.add_suffix('_match_median' ).reset_index() , how='left', on=['matchId'])
 imp_cols.extend(['damageDealt_match_median', 'killStreaks_match_median', 'walkDistance_match_median', 'killStreakrate_match_median', 'distance_over_weapons_match_median', 'total_time_by_distance_over_weapons_match_median', 'killPlace_over_total_time_by_distance_match_median', 'killPlace_over_totalDistance_match_median', 'killPlace_x_totalDistance_match_median', 'total_items_acquired_over_total_time_by_distance_match_median', 'total_items_acquired_norm_over_total_time_by_distance_match_median', 'kills_over_total_time_by_distance_match_median', 'kills_x_totalDistance_match_median', 'damageDealtNorm_match_median', 'L3_match_median', 'L3_over_total_time_by_distance_match_median', 'L3_x_total_time_by_distance_match_median', 'L3_over_totalDistance_match_median', 'L3_x_totalDistance_match_median'])
 df_out = df_out[imp_cols]
del tmp
gc.collect()
tmp = df.groupby(['matchId'])[features].agg('max')
 df_out = df_out.merge(tmp.add_suffix('_match_max' ).reset_index() , how='left', on=['matchId'])
 imp_cols.extend(['damageDealt_match_max', 'longestKill_match_max', 'rankPoints_match_max', 'rideDistance_match_max', 'swimDistance_match_max', 'walkDistance_match_max', 'heals_over_boosts_match_max', 'healthitems_norm_over_kills_match_max', 'walkDistance_over_heals_match_max', 'walkDistance_x_heals_match_max', 'walkDistance_over_kills_match_max', 'walkDistance_x_kills_match_max', 'walkDistance_over_healthitems_match_max', 'walkDistance_x_healthitems_match_max', 'total_time_by_distance_match_max', 'totalDistance_match_max', 'distance_over_weapons_match_max', 'distance_x_weapons_match_max', 'total_time_by_distance_over_weapons_match_max', 'total_time_by_distance_x_weapons_match_max', 'killPlace_over_total_time_by_distance_match_max', 'killPlace_x_total_time_by_distance_match_max', 'killPlace_over_totalDistance_match_max', 'killPlace_x_totalDistance_match_max', 'boosts_over_total_time_by_distance_match_max', 'boosts_x_total_time_by_distance_match_max', 'boosts_over_totalDistance_match_max', 'boosts_x_totalDistance_match_max', 'total_items_acquired_norm_match_max', 'total_items_acquired_over_total_time_by_distance_match_max', 'total_items_acquired_x_total_time_by_distance_match_max', 'total_items_acquired_norm_over_total_time_by_distance_match_max', 'heals_over_total_time_by_distance_match_max', 'heals_x_total_time_by_distance_match_max', 'heals_over_totalDistance_match_max', 'heals_x_totalDistance_match_max', 'kills_over_total_time_by_distance_match_max', 'kills_x_total_time_by_distance_match_max', 'kills_over_totalDistance_match_max', 'kills_x_totalDistance_match_max', 'killsNorm_match_max', 'damageDealtNorm_match_max', 'L3_match_max', 'points_match_max', 'L1_x_total_time_by_distance_match_max', 'L1_over_totalDistance_match_max', 'L2_over_total_time_by_distance_match_max', 'L2_x_total_time_by_distance_match_max', 'L2_over_totalDistance_match_max', 'L2_x_totalDistance_match_max', 'L3_over_total_time_by_distance_match_max', 'L3_x_total_time_by_distance_match_max', 'L3_over_totalDistance_match_max', 'L3_x_totalDistance_match_max'])
 df_out = df_out[imp_cols]
del tmp
gc.collect()
print("Step 7")
 tmp = df.groupby(['matchId','groupId'])[features].agg('sum')
 tmp2 = tmp.groupby('matchId')[features].rank(pct=True)
 df_out = df_out.merge(tmp.add_suffix('_sum' ).reset_index() , how='left', on=['matchId', 'groupId'])
 imp_cols.extend(['longestKill_sum', 'walkDistance_over_kills_sum', 'killPlace_over_maxPlace_sum', 'killPlace_over_numGroups_sum', 'damageDealtNorm_sum', 'maxPlaceNorm_sum', 'killPlace_over_playersJoined_sum', 'killPlacePerc_sum'])
 df_out = df_out[imp_cols]
df_out = df_out.merge(tmp2.add_suffix('_sum_rank' ).reset_index() , how='left', on=['matchId', 'groupId'])
 imp_cols.extend(['kills_sum_rank', 'killStreaks_sum_rank', 'longestKill_sum_rank', 'rankPoints_sum_rank', 'walkDistance_sum_rank', 'winPoints_sum_rank', 'killStreakrate_sum_rank', 'killPlace_over_maxPlace_sum_rank', 'total_time_by_distance_sum_rank', 'totalDistance_sum_rank', 'distance_x_weapons_sum_rank', 'total_time_by_distance_x_weapons_sum_rank', 'killPlace_over_total_time_by_distance_sum_rank', 'killPlace_x_total_time_by_distance_sum_rank', 'killPlace_over_totalDistance_sum_rank', 'boosts_x_total_time_by_distance_sum_rank', 'boosts_x_totalDistance_sum_rank', 'total_items_acquired_x_total_time_by_distance_sum_rank', 'total_items_acquired_norm_x_total_time_by_distance_sum_rank', 'kills_x_total_time_by_distance_sum_rank', 'killsNorm_sum_rank', 'killPlacePerc_sum_rank', 'L3_sum_rank', 'points_sum_rank'])
 df_out = df_out[imp_cols]
del tmp
del tmp2
gc.collect()
print("Step 8")
 df_out = df_out.assign(agg_group_size=df.groupby('groupId' ).groupId.transform('count'))
df_out = df_out.assign(agg_match_size=df.groupby('matchId' ).Id.transform('nunique'))
print("Step 9")
 del df_out["matchId"]
del df_out["groupId"]
reduce_mem_usage(df_out)
 gc.collect()
return df_out<prepare_x_and_y>
print('Train and test columns: {} {}'.format(len(train_df.columns), len(test_df.columns)) )
%%time
df = pd.read_csv(INPUT_DIR + 'train_V2.csv')
 df = df[df['maxPlace'] > 1]
reduce_mem_usage(df)
 target = 'winPlacePerc'
df = dataproc1(df)
 df_out = dataproc2(df)
 del df
gc.collect()
y_train = df_out[target]
x_train = df_out.drop(target, axis=1)
 del df_out
gc.collect()<train_model>
train = train_df.drop(columns=['ID_code','target'])
 train_label = train_df['target']
test = test_df.drop(columns='ID_code')
 test_ids = test_df['ID_code']
%%time
def run_lgb(train_X, train_y):
params = {"objective" : "regression", "metric" : "mae", 'n_estimators':50000,
"num_leaves" : 31, "learning_rate" : 0.05, "bagging_fraction" : 0.7,
"bagging_seed" : 0, "num_threads" : -1,"colsample_bytree" : 0.7
}
lgtrain = lgb.Dataset(train_X, label=train_y)
 model = lgb.train(params, lgtrain, verbose_eval=1)
 return model
m = run_lgb(x_train, y_train )<load_from_csv>
def del_punct(one_list):
return_list = []
regex = re.compile('['+re.escape("'")+']')
 for element in one_list:
return_list.append(regex.sub(" ",element ).strip())
 return return_list
df = pd.read_csv(INPUT_DIR + 'test_V2.csv')
 reduce_mem_usage(df)
 df = dataproc1(df)
 df_out = dataproc2(df, is_train='FALSE')
 y_pred = m.predict(df_out ).clip(0, 1 )<merge>
def distinguish_str(value_list):
output = []
regex = re.compile('[0-9]')
 for i,element in enumerate(value_list):
if regex.search(element):
output.append(float(element))
else:
output.append(element)
 return output
print("Preparing output file")
 long_file = df[['Id', 'matchId', 'groupId']]
short_file = df.groupby(['matchId','groupId'])['DBNOs'].agg('mean' ).reset_index()
short_file['winPlacePerc'] = y_pred
short_file = short_file[['matchId', 'groupId', 'winPlacePerc']]
merge_file = long_file.merge(short_file, on='groupId')
 output = merge_file.drop(['matchId_x', 'groupId', 'matchId_y'], axis=1)
 output = output.set_index('Id' )<define_variables>
def model_gbm(train,train_label,test,test_ids,nfolds=5,hyperparameters=None):
feature_names = list(train.columns)
 valid_scores = np.zeros(len(train))
predictions = np.zeros(len(test))
feature_importance_df = pd.DataFrame()
max_iters_df = pd.DataFrame(columns=["folds","iters"])
 iters = []
folds = []
if hyperparameters:
params = hyperparameters
del(params['n_estimators'])
 params['boost_from_average'] = True
params['seed'] = 31452
params['feature_fraction_seed'] = 31415
params['bagging_seed'] = 31415
params['drop_seed'] = 31415
params['data_random_seed'] =31415
params['metric'] = 'auc'
else:
params = {
'num_leaves': 6,
'max_bin': 63,
'min_data_in_leaf': 45,
'learning_rate': 0.01,
'min_sum_hessian_in_leaf': 0.000446,
'bagging_fraction': 0.55,
'bagging_freq': 5,
'max_depth': 14,
'save_binary': True,
'seed': 31452,
'feature_fraction_seed': 31415,
'feature_fraction': 0.51,
'bagging_seed': 31415,
'drop_seed': 31415,
'data_random_seed': 31415,
'objective': 'binary',
'boosting_type': 'gbdt',
'verbose': 1,
'metric': 'auc',
'is_unbalance': True,
'boost_from_average': False
}
strfkold = StratifiedKFold(n_splits=nfolds,shuffle=True,random_state=12)
 for i,(train_indices,valid_indices)in enumerate(strfkold.split(train.values,train_label.values)) :
print("{} fold processing".format(i+1)+"
d_train = lgb.Dataset(train.values[train_indices,:],label = train_label[train_indices])
 d_valid = lgb.Dataset(train.values[valid_indices,:],label = train_label[valid_indices])
 n_rounds = 15000
lgb_model = lgb.train(params,d_train,num_boost_round=n_rounds,valid_sets=[d_train,d_valid],valid_names=['train','valid'],verbose_eval=1000,early_stopping_rounds=250)
 valid_scores[valid_indices] = lgb_model.predict(train.values[valid_indices,:],num_iteration=lgb_model.best_iteration)
 fold_importance_df = pd.DataFrame(columns=["Feature","importance","fold"])
 fold_importance_df["Feature"] = feature_names
fold_importance_df["importance"] = lgb_model.feature_importance()
fold_importance_df["fold"] = i + 1
feature_importance_df = pd.concat([feature_importance_df,fold_importance_df],axis=0)
 folds.append(i+1)
 iters.append(lgb_model.best_iteration)
 predictions += lgb_model.predict(test.values,num_iteration=lgb_model.best_iteration)/nfolds
display("valid_set score is %f and best_iteration is %d of %d fold"%(roc_auc_score(train_label[valid_indices],valid_scores[valid_indices]),lgb_model.best_iteration,i+1))
max_iters_df["folds"] = folds
max_iters_df["iters"] = iters
display("CV score of valid_set for %d fold is %f and maximum of best_iteration is %d of %d fold"%(nfolds,roc_auc_score(train_label,valid_scores),max_iters_df['iters'].max() ,max_iters_df['iters'].idxmax() +1))
return valid_scores,predictions,feature_importance_df
df_test = df
df_sub = output<merge>
from hyperopt import hp,tpe,Trials,fmin, STATUS_OK
from hyperopt.pyll.stochastic import sample
df_sub = df_sub.merge(df_test[["Id", "matchId", "groupId", "maxPlace", "numGroups"]], on="Id", how="left")
 df_sub_group = df_sub.groupby(["matchId", "groupId"] ).first().reset_index()
df_sub_group["rank"] = df_sub_group.groupby(["matchId"])["winPlacePerc"].rank()
df_sub_group = df_sub_group.merge(
 df_sub_group.groupby("matchId")["rank"].max().to_frame("max_rank" ).reset_index() ,
on="matchId", how="left")
 df_sub_group["adjusted_perc"] =(df_sub_group["rank"] - 1)/(df_sub_group["numGroups"] - 1)
 df_sub = df_sub.merge(df_sub_group[["adjusted_perc", "matchId", "groupId"]], on=["matchId", "groupId"], how="left")
 df_sub["winPlacePerc"] = df_sub["adjusted_perc"]
df_sub.loc[df_sub.maxPlace == 0, "winPlacePerc"] = 0
df_sub.loc[df_sub.maxPlace == 1, "winPlacePerc"] = 1
subset = df_sub.loc[df_sub.maxPlace > 1]
gap = 1.0 /(subset.maxPlace.values - 1)
 new_perc = np.around(subset.winPlacePerc.values / gap)* gap
df_sub.loc[df_sub.maxPlace > 1, "winPlacePerc"] = new_perc
df_sub.loc[(df_sub.maxPlace > 1)&(df_sub.numGroups == 1), "winPlacePerc"] = 0
assert df_sub["winPlacePerc"].isnull().sum() == 0 
<save_to_csv>
import csv
import ast
from timeit import default_timer as timer
df_sub[["Id", "winPlacePerc"]].to_csv("submission.csv", index=False )<string_transform>
def lgb_roc_auc(labels,predictions):
metric_value = roc_auc_score(labels,predictions)
 return 'roc_auc',metric_value,True
def take_part_of_data(df, part):
match_ids = df['matchId'].unique()
match_ids_part = np.random.choice(match_ids, int(part * len(match_ids)))
 df = df[df['matchId'].isin(match_ids_part)]
del match_ids
del match_ids_part<feature_engineering>
def objective(hyperparameters, nfold=5):
global ITERATION
ITERATION += 1
for parameter_name in ['max_depth','num_leaves','subsample_for_bin','min_child_samples','subsample_freq']:
hyperparameters[parameter_name] = int(hyperparameters[parameter_name])
 strkfold = StratifiedKFold(n_splits=nfold,shuffle=True)
 features = np.array(train)
 labels = np.array(train_label ).reshape(( -1))
valid_scores = []
best_estimators = []
run_times = []
model = lgb.LGBMClassifier(**hyperparameters,n_jobs=-1,metric='None',n_estimators=1000)
 for i,(train_indices,valid_indices)in enumerate(strkfold.split(features,labels)) :
print("
X_train,X_valid = features[train_indices],features[valid_indices]
y_train,y_valid = labels[train_indices], labels[valid_indices]
start = timer()
model.fit(X_train,y_train,early_stopping_rounds=50,
eval_metric=lgb_roc_auc,eval_set=[(X_train,y_train),(X_valid,y_valid)],
eval_names=['train','valid'],verbose=200)
 end = timer()
valid_scores.append(model.best_score_['valid']['roc_auc'])
 best_estimators.append(model.best_iteration_)
 run_times.append(end-start)
 score = np.mean(valid_scores)
 score_std = np.std(valid_scores)
 loss = 1-score
run_time = np.mean(run_times)
 run_time_std = np.std(run_times)
 estimators = int(np.mean(best_estimators))
hyperparameters['n_estimators'] = estimators
of_connection = open(OUT_FILE,'a')
 writer = csv.writer(of_connection)
 writer.writerow([loss,hyperparameters,ITERATION,run_time,score,score_std])
 of_connection.close()
display(f'Iteration: {ITERATION}, Score: {round(score, 4)}.')
 if ITERATION % PROGRESS == 0:
display(f'Iteration: {ITERATION}, Current Score: {round(score, 4)}.')
 return {'loss': loss, 'hyperparameters': hyperparameters, 'iteration': ITERATION,
'time': run_time, 'time_std': run_time_std, 'status': STATUS_OK,
'score': score, 'score_std': score_std}
def add_new_features_1(df):
df['totalDistance'] = df['rideDistance'] + df['walkDistance'] + df['swimDistance']
df['healsAndBoosts'] = df['heals'] + df['boosts']
df['headshotKillsOverKills'] = df['headshotKills'] / df['kills']
df['headshotKillsOverKills'].fillna(0, inplace=True)
 df['killStreaksOverKills'] = df['killStreaks'] / df['kills']
df['killStreaksOverKills'].fillna(0, inplace=True)
 df['killsAndAssists'] = df['kills'] + df['assists']
df['assistsAndRevives'] = df['assists'] + df['revives']<feature_engineering>
space = {
'boosting_type':'gbdt',
'objective':'binary',
'is_unbalance':True,
'subsample': hp.uniform('gbdt_subsample',0.5,1),
'subsample_freq':hp.quniform('gbdt_subsample_freq',1,10,1),
'max_depth': hp.quniform('max_depth',5,20,3),
'num_leaves': hp.quniform('num_leaves',20,60,10),
'learning_rate':hp.loguniform('learning_rate',np.log(0.025),np.log(0.25)) ,
'subsample_for_bin':hp.quniform('subsample_for_bin',2000,100000,2000),
'min_child_samples': hp.quniform('min_child_samples',5,80,5),
'colsample_bytree': hp.uniform('colsample_by_tree', 0.5, 1.0),
'min_child_weight':hp.uniform('min_child_weight',0.01,0.000001)
 }
def add_new_features_2(df):
df['playersJoined'] = df.groupby('matchId')['matchId'].transform('count')
 df['killsAndAssistsOverPlayersJoined'] = df['killsAndAssists'] *(( 100 - df['playersJoined'])/ 100 + 1)
 df['matchDurationOverPlayersJoined'] = df['matchDuration'] *(( 100 - df['playersJoined'])/ 100 + 1)
 df['damageDealtOverPlayersJoined'] = df['damageDealt'] *(( 100 - df['playersJoined'])/ 100 + 1 )<feature_engineering>
algo = tpe.suggest
def add_new_features_3(df):
df['totalDistanceOverKillsAndAssists'] = df['totalDistance'] / df['killsAndAssists']
df['totalDistanceOverKillsAndAssists'].fillna(0, inplace=True)
 df['totalDistanceOverKillsAndAssists'].replace(np.inf, 0, inplace=True)
 df['totalDistanceOverHealsAndBoosts'] = df['totalDistance'] / df['healsAndBoosts']
df['totalDistanceOverHealsAndBoosts'].fillna(0, inplace=True)
 df['totalDistanceOverHealsAndBoosts'].replace(np.inf, 0, inplace=True )<feature_engineering>
trials = Trials()
OUT_FILE = 'optimization.csv'
of_connection = open(OUT_FILE, 'w')
 writer = csv.writer(of_connection)
 MAX_EVALS = 10
PROGRESS = 10
ITERATION = 0
headers = ['loss', 'hyperparameters', 'iteration', 'runtime', 'score', 'std']
writer.writerow(headers)
 of_connection.close()
def add_new_features_4(df):
df['headshotRate'] = df['kills'] / df['headshotKills']
df['killStreakRate'] = df['killStreaks'] / df['kills']
df['healsAndBoosts'] = df['heals'] + df['boosts']
df['totalDistance'] = df['rideDistance'] + df['walkDistance'] + df['swimDistance']
df['killPlaceOverMaxPlace'] = df['killPlace'] / df['maxPlace']
df['headshotKillsOverKills'] = df['headshotKills'] / df['kills']
df['distanceOverWeapons'] = df['totalDistance'] / df['weaponsAcquired']
df['walkDistanceOverHeals'] = df['walkDistance'] / df['heals']
df['walkDistanceOverKills'] = df['walkDistance'] / df['kills']
df['killsPerWalkDistance'] = df['kills'] / df['walkDistance']
df["skill"] = df['headshotKills'] + df['roadKills']
df[df == np.Inf] = np.NaN
df[df == np.NINF] = np.NaN
df.fillna(0, inplace=True )<prepare_x_and_y>
print("beginning time is {}".format(datetime.datetime.now()))
display("Running Optimization for {} Trials.".format(MAX_EVALS))
best = fmin(fn = objective, space = space, algo = tpe.suggest, trials = trials,max_evals = MAX_EVALS)
 print("end time is {}".format(datetime.datetime.now()))
def feature_engineering(df, is_train=True):
df['rankPoints'] = np.where(df['rankPoints'] <= 0, 0, df['rankPoints'])
 features = list(df.columns)
 features.remove("matchId")
 features.remove("groupId")
 features.remove("matchDuration")
 features.remove("matchType")
 if 'winPlacePerc' in features:
features.remove('winPlacePerc')
 y = None
if is_train:
y = df.groupby(['matchId','groupId'])['winPlacePerc'].agg('mean')
 elif 'winPlacePerc' in df.columns:
y = df['winPlacePerc']
agg = df.groupby(['matchId','groupId'])[features].agg('mean')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
if is_train:
df_out = agg.reset_index() [['matchId','groupId']]
else:
df_out = df[['matchId','groupId']]
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_mean", "_mean_rank"], how='left', on=['matchId', 'groupId'])
 agg = df.groupby(['matchId','groupId'])[features].agg('max')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_max", "_max_rank"], how='left', on=['matchId', 'groupId'])
 agg = df.groupby(['matchId','groupId'])[features].agg('min')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_min", "_min_rank"], how='left', on=['matchId', 'groupId'])
 agg = df.groupby(['matchId','groupId'] ).size().reset_index(name='group_size')
 df_out = df_out.merge(agg, how='left', on=['matchId', 'groupId'])
 agg = df.groupby(['matchId'])[features].agg('mean' ).reset_index()
df_out = df_out.merge(agg, suffixes=["", "_match_mean"], how='left', on=['matchId'])
 agg = df.groupby(['matchId'] ).size().reset_index(name='match_size')
 df_out = df_out.merge(agg, how='left', on=['matchId'])
 df_out.drop(["matchId", "groupId"], axis=1, inplace=True)
 del agg, agg_rank
return df_out, y<predict_on_test>
hyperparameters = results.hyperparameters[0]
class Estimator(object):
def fit(self, x_train, y_train, x_valid, y_valid):
raise NotImplementedException
def predict(self, x):
raise NotImplementedException<train_model>
hyperparameters = {
'boosting_type': 'gbdt',
'colsample_bytree': 0.7812943473676428,
'is_unbalance': True,
'learning_rate': 0.012732207618246335,
'max_bin': 200,
'max_depth': 14,
'min_child_samples': 70,
'min_child_weight': 0.0010242091278688855,
'num_leaves': 10,
'objective': 'binary',
'subsample': 0.8026192939361728,
'subsample_for_bin': 72000,
'subsample_freq': 7,
'n_estimators': 6589}
class ScikitLearnEstimator(Estimator):
def __init__(self, estimator):
self.estimator = estimator
def fit(self, x_train, y_train, x_valid, y_valid):
self.estimator.fit(x_train, y_train)
 def predict(self, x):
return self.estimator.predict(x )<train_model>
val_scores, predictions, gbm_fi= model_gbm(train,train_label,test,test_ids,hyperparameters=hyperparameters )
<predict_on_test><EOS> 
submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['target'] = predictions
submission.to_csv("submission.csv",index=False )