Kaggle competition: Google Cloud & NCAA® ML Competition 2019-Women's
<SOS> metric: LogLoss Kaggle data source: womens-machine-learning-competition-2019<feature_engineering>
%matplotlib inline
InteractiveShell.ast_node_interactivity = "all"
all_set['Surname'] = all_set['Name'].apply(lambda x: x.split(',')[0].strip() )<feature_engineering>
data_dir = '.. /input/stage2wdatafiles/'
df_seed = pd.read_csv(data_dir + 'WNCAATourneySeeds.csv')
 df_result = pd.read_csv(data_dir + 'WNCAATourneyCompactResults.csv')
 df_seed.tail(3)
 df_result.tail(3 )
all_set['FamilySurvival'] = 0.5
for surname in all_set['Surname'].unique() :
df = all_set[all_set['Surname'] == surname]
if df.shape[0] > 1:
smin = df['Survived'].min()
smax = df['Survived'].max()
for idx, row in df.iterrows() :
passengerid = row['PassengerId']
if smax == 1.0:
all_set.loc[all_set['PassengerId'] == passengerid, 'FamilySurvival'] = 1.0
elif smin == 0.0:
all_set.loc[all_set['PassengerId'] == passengerid, 'FamilySurvival'] = 0.0<count_values>
def seed_to_int(seed):
s_int = int(seed[1:3])
 return s_int
def clean_df(df_seed, df_result):
df_seed['seed_int'] = df_seed['Seed'].apply(seed_to_int)
 df_seed.drop(['Seed'], axis=1, inplace=True)
 df_result.drop(['DayNum', 'WLoc', 'NumOT'], axis=1, inplace=True)
 return df_seed, df_result
df_seed, df_result = clean_df(df_seed, df_result)
 df_seed.head(3)
 df_result.head(3 )
all_set['Embarked'].value_counts()<feature_engineering>
def merge_seed_result(df_seed, df_result):
df_win_seed = df_seed.rename(columns={'TeamID':'WTeamID', 'seed_int':'WSeed'})
 df_loss_seed = df_seed.rename(columns={'TeamID':'LTeamID', 'seed_int':'LSeed'})
 df_result = df_result.merge(df_win_seed, how='left', on=['Season', 'WTeamID'])
 df_result = df_result.merge(df_loss_seed, how='left', on=['Season', 'LTeamID'])
 df_result['SeedDiff'] = np.abs(df_result['WSeed'] - df_result['LSeed'])
 df_result['ScoreDiff'] = np.abs(df_result['WScore'] - df_result['LScore'])
 return df_result
df_result = merge_seed_result(df_seed, df_result)
 df_result.head(3 )
all_set['Embarked'] = all_set['Embarked'].fillna('S' )<count_missing_values>
df_result = df_result[df_result['ScoreDiff']>3]
all_set[all_set['Fare'].isna() ]<data_type_conversions>
df_result['upset'] = [1 if ws > ls else 0 for ws, ls, in zip(df_result["WSeed"], df_result["LSeed"])]
print("upset probability")
 df_result['upset'].value_counts() / len(df_result)* 100
all_set['Fare'].fillna(all_set[all_set['Embarked'] == 'S']['Fare'].mean() , inplace=True )<feature_engineering>
this_season=2019
total_season=10
train = df_result[(df_result["Season"]>=(this_season - total_season)) &(df_result["Season"]<(this_season-1)) ]
print(train.shape )
all_set['Deck'] = all_set['Cabin'].apply(lambda x: x[0] if type(x)== str else '' )<filter>
df_result["Seed_combi"]=[str(ws)+'_'+str(ls)if ws<ls else str(ls)+'_'+str(ws)for ws, ls in zip(df_result["WSeed"], df_result["LSeed"])]
df_result.head(3 )
all_set[all_set['Deck'] == 'G']<feature_engineering>
df_result_aggs = pd.DataFrame()
df_result_filter_aggs = pd.DataFrame()
df_result_season = df_result[(df_result["Season"]>=(this_season - total_season)) &(df_result["Season"]<(this_season-1)) ]
for value in range(16):
df_result_agg = df_result_season[df_result_season["SeedDiff"]==value].groupby("SeedDiff" ).agg({"upset": ["mean", "count"]})
 df_result_agg.columns = [col[0]+"_"+col[1]+"_"+"all" for col in df_result_agg.columns]
df_result_filter_agg = df_result_season[df_result_season["SeedDiff"]==value].groupby("Seed_combi" ).agg({"upset": ["mean", "count"]})
 df_result_filter_agg.columns = [col[0]+"_"+col[1] for col in df_result_filter_agg.columns]
if value==0:
df_result_agg["upset_mean_all"] = 0.5
df_result_filter_agg["upset_mean"] = 0.5
df_result_aggs = pd.concat([df_result_aggs, df_result_agg])
 df_result_filter_aggs = pd.concat([df_result_filter_aggs, df_result_filter_agg])
 df_result_aggs
df_result_filter_aggs.tail(10 )
deck_fare = {}
for deck in all_set['Deck'].unique() :
if len(deck)== 1 and deck in 'ABCDEF':
deck_fare[deck] = all_set[ 
(all_set['Cabin'].apply(lambda x: True if type(x)== str else False)) & 
(all_set['Deck'] == deck)
 ]['Fare'].mean()
deck_fare<feature_engineering>
df_result = df_result.join(df_result_aggs, how='left', on="SeedDiff" ).join(df_result_filter_aggs, how='left', on='Seed_combi')
 df_result["upset_prob"] = [m if c > 20 else a for a, m, c in zip(df_result["upset_mean_all"], df_result["upset_mean"], df_result["upset_count"])]
df_result.tail()
def find_deck(fare):
dist = 1000
res = 'F'
for key in deck_fare.keys() :
new_dist = np.abs(fare - deck_fare[key])
 if new_dist < dist:
dist = new_dist
res = key
return res
all_set.loc[all_set['Cabin'].isna() , 'Deck'] = all_set['Fare'].apply(find_deck )<feature_engineering>
valid = df_result[(df_result["Season"]==(this_season-1)) ]
log_loss(valid['upset'], valid['upset_prob'] )
all_set['Family'] = 1 + all_set['SibSp'] + all_set['Parch']
all_set['Alone'] = all_set['Family'].apply(lambda x: 1 if x == 1 else 0 )<feature_engineering>
valid = df_result[(df_result["Season"]==(this_season-1)) ]
valid = valid.join(df_result_aggs.drop("upset_count_all", axis=1), how='left', on='SeedDiff')
 valid.fillna(0, inplace=True)
 log_loss(valid['upset'], valid['upset_prob_manually'] )
age_by_title = {}
for title in all_set['Title'].unique() :
age_by_title[title] = all_set[ 
(all_set['Age'].apply(lambda x: True if type(x)== float else False)) & 
(all_set['Title'] == title)
 ]['Age'].mean()
age_by_title<feature_engineering>
log_loss(valid['upset'], np.clip(valid['upset_prob_manually'], 0.05, 0.95))
all_set.loc[all_set['Age'].isna() , 'Age'] = all_set['Title'].apply(lambda x: age_by_title[x] )<categorify>
df_seed_2019 = df_seed[df_seed["Season"]==2019]
all_set['AgeBin'] = pd.qcut(all_set['Age'], 5)
 all_set['AgeCode'] = all_set['AgeBin']
all_set = label_encode(all_set, 'AgeCode' )<feature_engineering>
this_season=2019
total_season=10
train = df_result[(df_result["Season"]>=(this_season - total_season)) ]
print(train.shape)
 df_result["Seed_combi"]=[str(ws)+'_'+str(ls)if ws<ls else str(ls)+'_'+str(ws)for ws, ls in zip(df_result["WSeed"], df_result["LSeed"])]
df_result.head()
all_set['AgeDist'] = 1/ np.exp(np.abs(all_set['Age'] - all_set[all_set['Survived'] == 1]['Age'].median()))<categorify>
df_result_aggs = pd.DataFrame()
df_result_filter_aggs = pd.DataFrame()
for value in range(16):
df_result_agg = df_result[df_result["SeedDiff"]==value].groupby("SeedDiff" ).agg({"upset": ["mean", "count"]})
 df_result_agg.columns = [col[0]+"_"+col[1]+"_"+"all" for col in df_result_agg.columns]
df_result_filter_agg = df_result[df_result["SeedDiff"]==value].groupby("Seed_combi" ).agg({"upset": ["mean", "count"]})
 df_result_filter_agg.columns = [col[0]+"_"+col[1] for col in df_result_filter_agg.columns]
if value==0:
df_result_agg["upset_mean_all"] = 0.5
df_result_filter_agg["upset_mean"] = 0.5
df_result_aggs = pd.concat([df_result_aggs, df_result_agg])
 df_result_filter_aggs = pd.concat([df_result_filter_aggs, df_result_filter_agg])
 df_result_aggs
df_result_filter_aggs.tail(10 )
all_set = label_encode(all_set, 'Sex')
 compare(all_set, 'Sex' )<categorify>
df_result_aggs.loc[0, 'upset_mean_all'] = 0.5
df_result_aggs.loc[10, 'upset_mean_all'] =(0.0 + df_result_aggs.loc[11, 'upset_mean_all'])/ 2
df_result_aggs.loc[11, 'upset_mean_all'] =(0.0 + df_result_aggs.loc[15, 'upset_mean_all'])/ 2
df_result_aggs.loc[12, 'upset_mean_all'] =(0.0 + df_result_aggs.loc[15, 'upset_mean_all'])/ 2
df_result_aggs.loc[13, 'upset_mean_all'] =(0.0 + df_result_aggs.loc[15, 'upset_mean_all'])/ 2
df_result_aggs.loc[14, 'upset_mean_all'] =(0.0 + df_result_aggs.loc[15, 'upset_mean_all'])/ 2
df_result_aggs = df_result_aggs.fillna(-1)
 sns.barplot(df_result_aggs.index, df_result_aggs.upset_mean_all)
 plt.title('probability of upset based on past result aggretation')
 plt.show()
all_set = label_encode(all_set, 'Embarked')
 compare(all_set, 'Embarked' )<categorify>
test = pd.read_csv(".. /input/WSampleSubmissionStage2.csv")
 test = pd.DataFrame(np.array([ID.split("_")for ID in test["ID"]]), columns=["Season", "TeamA", "TeamB"], dtype=int)
 test.head(3)
 test = test.merge(df_seed_2019, how='left', left_on=["Season", "TeamA"], right_on=["Season", "TeamID"])
 test = test.rename(columns={"seed_int": "TeamA_seed"} ).drop("TeamID", axis=1)
 test = test.merge(df_seed_2019, how='left', left_on=["Season", "TeamB"], right_on=["Season", "TeamID"])
 test = test.rename(columns={"seed_int": "TeamB_seed"} ).drop("TeamID", axis=1)
 test['SeedDiff'] = np.abs(test.TeamA_seed - test.TeamB_seed)
 test.head(3 )
all_set = label_encode(all_set, 'Deck')
 compare(all_set, 'Deck' )<count_values>
test["Seed_combi"]=[str(a)+'_'+str(b)if a<b else str(b)+'_'+str(a)for a, b in zip(test["TeamA_seed"], test["TeamB_seed"])]
test.head()
test = test.join(df_result_aggs, how='left', on="SeedDiff" ).join(df_result_filter_aggs, how='left', on='Seed_combi' ).fillna(-1)
 test["upset_prob"] = [m if c > 20 else a for a, m, c in zip(test["upset_mean_all"], test["upset_mean"], test["upset_count"])]
test["win_prob"] = [(1-upset_prob)if teamA<teamB else upset_prob if teamA>teamB else 0.5
for teamA, teamB, upset_prob in zip(test['TeamA_seed'], test['TeamB_seed'], test['upset_prob'])]
test.tail()
all_set['Title'].value_counts()<count_unique_values>
submit = pd.read_csv(".. /input/WSampleSubmissionStage2.csv")
 submit["Pred"] = test['win_prob']
submit.to_csv("submission_agg_all_manually_noclip.csv", index=False)
 submit.head()
<categorify><EOS> 
clipped_sub = np.clip(test['win_prob'], 0.05, 0.95)
 submit = pd.read_csv(".. /input/WSampleSubmissionStage2.csv")
 submit["Pred"] = clipped_sub
submit.to_csv("submission_agg_all_manually_cliped.csv", index=False)
 submit.head()