Kaggle competition: Google Cloud & NCAA® ML Competition 2020-NCAAW
<SOS> metric: logloss Kaggle data source: google-cloud-ncaa-march-madness-2020-division-1-womens-tournament<categorify>
finish_data = 2014
lgb_num_leaves_max = 200
lgb_in_leaf = 10
lgb_lr = 0.001
lgb_bagging = 7
xgb_max_depth = 7
xgb_min_child_weight = 75
xgb_lr = 0.0004
xgb_num_boost_round_max = 3000
w_lgb = 0.4
w_xgb = 0.5
w_logreg = 1 - w_lgb - w_xgb
w_logreg
dataset_test=tf.data.Dataset.from_tensor_slices(( Xids_test,Xmask_test))
def map_func(input_ids,mask):
return {'input_ids':input_ids,'attention_mask':mask}
dataset_test=dataset_test.map(map_func)
 dataset_test=dataset_test.batch(64 ).prefetch(1000 )<choose_model_class>
warnings.filterwarnings("ignore" )
distil_bert = 'distilbert-base-uncased'
config = DistilBertConfig(dropout=0.4, attention_dropout=0.4)
 config.output_hidden_states = False
transformer_model = TFDistilBertModel.from_pretrained(distil_bert, config = config)
 input_ids_in = tf.keras.layers.Input(shape=(SEQ_length,), name='input_ids', dtype='int32')
 input_masks_in = tf.keras.layers.Input(shape=(SEQ_length,), name='attention_mask', dtype='int32')
 embedding_layer = transformer_model(input_ids_in, attention_mask=input_masks_in)[0]
X = tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(100, return_sequences=True, dropout=0.4, recurrent_dropout=0.4))(embedding_layer)
 X = tf.keras.layers.GlobalMaxPool1D()(X)
 X = tf.keras.layers.Dense(64, activation='relu' )(X)
 X = tf.keras.layers.Dropout(0.3 )(X)
 X = tf.keras.layers.Dense(32, activation='relu' )(X)
 X = tf.keras.layers.Dropout(0.3 )(X)
 X = tf.keras.layers.Dense(1, activation='sigmoid' )(X)
 model = tf.keras.Model(inputs=[input_ids_in, input_masks_in], outputs = X)
 for layer in model.layers[:3]:
layer.trainable = False 
<choose_model_class>
tourney_result = pd.read_csv('.. /input/google-cloud-ncaa-march-madness-2020-division-1-womens-tournament/WDataFiles_Stage1/WNCAATourneyCompactResults.csv')
 if finish_data == 2014:
tourney_result = tourney_result[tourney_result['Season'] < 2015]
tourney_seed = pd.read_csv('.. /input/google-cloud-ncaa-march-madness-2020-division-1-womens-tournament/WDataFiles_Stage1/WNCAATourneySeeds.csv')
 if finish_data == 2014:
tourney_seed = tourney_seed[tourney_seed['Season'] < 2015]
tourney_result = tourney_result.drop(['DayNum', 'WScore', 'LScore', 'WLoc', 'NumOT'], axis=1)
 tourney_result = pd.merge(tourney_result, tourney_seed, left_on=['Season', 'WTeamID'], right_on=['Season', 'TeamID'], how='left')
 tourney_result.rename(columns={'Seed':'WSeed'}, inplace=True)
 tourney_result = tourney_result.drop('TeamID', axis=1)
 tourney_result = pd.merge(tourney_result, tourney_seed, left_on=['Season', 'LTeamID'], right_on=['Season', 'TeamID'], how='left')
 tourney_result.rename(columns={'Seed':'LSeed'}, inplace=True)
 tourney_result = tourney_result.drop('TeamID', axis=1 )
model.compile(loss=tf.keras.losses.BinaryCrossentropy() ,
optimizer='adam',metrics=[tf.keras.metrics.AUC() ,tf.keras.metrics.Precision() ,tf.keras.metrics.Recall()
] )<train_model>
def get_seed(x):
return int(x[1:3])
 tourney_result['WSeed'] = tourney_result['WSeed'].map(lambda x: get_seed(x))
tourney_result['LSeed'] = tourney_result['LSeed'].map(lambda x: get_seed(x))
history=model.fit(train,validation_data=val,epochs=10 )<save_to_csv>
season_result = pd.read_csv('.. /input/google-cloud-ncaa-march-madness-2020-division-1-womens-tournament/WDataFiles_Stage1/WRegularSeasonCompactResults.csv')
 if finish_data == 2014:
season_result = season_result[season_result['Season'] < 2015]
season_win_result = season_result[['Season', 'WTeamID', 'WScore']]
season_lose_result = season_result[['Season', 'LTeamID', 'LScore']]
season_win_result.rename(columns={'WTeamID':'TeamID', 'WScore':'Score'}, inplace=True)
 season_lose_result.rename(columns={'LTeamID':'TeamID', 'LScore':'Score'}, inplace=True)
 season_result = pd.concat(( season_win_result, season_lose_result)).reset_index(drop=True)
 season_score = season_result.groupby(['Season', 'TeamID'])['Score'].sum().reset_index()
predictions=model.predict(dataset_test)
 df_test['label']=predictions
df_test.to_csv('submission.csv',columns=['urlid','label'],index=False )<categorify>
tourney_result = pd.merge(tourney_result, season_score, left_on=['Season', 'WTeamID'], right_on=['Season', 'TeamID'], how='left')
 tourney_result.rename(columns={'Score':'WScoreT'}, inplace=True)
 tourney_result = tourney_result.drop('TeamID', axis=1)
 tourney_result = pd.merge(tourney_result, season_score, left_on=['Season', 'LTeamID'], right_on=['Season', 'TeamID'], how='left')
 tourney_result.rename(columns={'Score':'LScoreT'}, inplace=True)
 tourney_result = tourney_result.drop('TeamID', axis=1 )
input_x=tf.data.Dataset.from_tensor_slices(( Xids,Xmask,y))
def map_func(input_ids,mask,labels):
return {'input_ids':input_ids,'attention_mask':mask}
input_x=input_x.map(map_func)
 input_x=input_x.shuffle(100000 ).batch(32 ).prefetch(1000)
 y_true = y
y_true<predict_on_test>
tourney_win_result = tourney_result.drop(['Season', 'WTeamID', 'LTeamID'], axis=1)
 tourney_win_result.rename(columns={'WSeed':'Seed1', 'LSeed':'Seed2', 'WScoreT':'ScoreT1', 'LScoreT':'ScoreT2'}, inplace=True )
y_pred=model.predict(dataset)
 y_pred
y_pred = np.round(y_pred)
 y_pred
print(metrics.classification_report(y_true, y_pred))<import_modules>
tourney_lose_result = tourney_win_result.copy()
tourney_lose_result['Seed1'] = tourney_win_result['Seed2']
tourney_lose_result['Seed2'] = tourney_win_result['Seed1']
tourney_lose_result['ScoreT1'] = tourney_win_result['ScoreT2']
tourney_lose_result['ScoreT2'] = tourney_win_result['ScoreT1']
import os
from pathlib import Path
import numpy as np
import pandas as pd
from sklearn.preprocessing import MinMaxScaler
from sklearn.model_selection import cross_validate, GridSearchCV
from sklearn.pipeline import make_pipeline
from sklearn.ensemble import RandomForestClassifier, VotingClassifier
from sklearn.svm import SVC
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import accuracy_score<define_variables>
tourney_win_result['Seed_diff'] = tourney_win_result['Seed1'] - tourney_win_result['Seed2']
tourney_win_result['ScoreT_diff'] = tourney_win_result['ScoreT1'] - tourney_win_result['ScoreT2']
tourney_lose_result['Seed_diff'] = tourney_lose_result['Seed1'] - tourney_lose_result['Seed2']
tourney_lose_result['ScoreT_diff'] = tourney_lose_result['ScoreT1'] - tourney_lose_result['ScoreT2']
FILEDIR = Path('/kaggle/input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament' )<load_from_csv>
tourney_win_result['result'] = 1
tourney_lose_result['result'] = 0
train_df = pd.concat(( tourney_win_result, tourney_lose_result)).reset_index(drop=True)
 train_df
sub = pd.read_csv(FILEDIR / 'MSampleSubmissionStage1_2020.csv', usecols=['ID'])
 id_splited = sub['ID'].str.split('_', expand=True ).astype(int ).rename(columns={0: 'Season', 1: 'Team1', 2: 'Team2'})
 sub = pd.concat([sub, id_splited], axis=1 ).set_index(['Season', 'Team1', 'Team2'] ).sort_index()<count_duplicates>
test_df = pd.read_csv('.. /input/google-cloud-ncaa-march-madness-2020-division-1-womens-tournament/WSampleSubmissionStage1_2020.csv')
 sub = test_df.copy()
tourney_teams = {}
tourney_teams_all = set()
for season in sub.index.get_level_values('Season' ).drop_duplicates() :
tourney_teams[season] = set()
tourney_teams[season].update(sub.loc[season].index.get_level_values('Team1'))
tourney_teams[season].update(sub.loc[season].index.get_level_values('Team2'))
tourney_teams_all.update(tourney_teams[season])
 {k: len(v)for k, v in tourney_teams.items() }<load_from_csv>
test_df['Season'] = test_df['ID'].map(lambda x: int(x[:4]))
test_df['WTeamID'] = test_df['ID'].map(lambda x: int(x[5:9]))
test_df['LTeamID'] = test_df['ID'].map(lambda x: int(x[10:14]))
conferences = pd.read_csv(FILEDIR / 'MDataFiles_Stage1/MTeamConferences.csv')
 conferences = pd.concat(
 [conferences.query('Season == @season and TeamID in @teams')for season, teams in tourney_teams.items() ])
 conferences = conferences.set_index(['Season', 'TeamID'] ).sort_index()<load_from_csv>
tourney_seed = pd.read_csv('.. /input/google-cloud-ncaa-march-madness-2020-division-1-womens-tournament/WDataFiles_Stage1/WNCAATourneySeeds.csv')
 if finish_data == 2014:
tourney_seed = tourney_seed[tourney_seed['Season'] > 2014]
coaches = pd.read_csv(FILEDIR / 'MDataFiles_Stage1/MTeamCoaches.csv')
 coaches = pd.concat(
 [coaches.query('Season == @season and TeamID in @team')for season, team in tourney_teams.items() ])
 coaches = coaches[coaches['LastDayNum'] == 154].set_index(['Season', 'TeamID'] ).sort_index() [['CoachName']]<load_from_csv>
season_result = pd.read_csv('.. /input/google-cloud-ncaa-march-madness-2020-division-1-womens-tournament/WDataFiles_Stage1/WRegularSeasonCompactResults.csv')
 if finish_data == 2014:
season_result = season_result[season_result['Season'] > 2014]
season_win_result = season_result[['Season', 'WTeamID', 'WScore']]
season_lose_result = season_result[['Season', 'LTeamID', 'LScore']]
season_win_result.rename(columns={'WTeamID':'TeamID', 'WScore':'Score'}, inplace=True)
 season_lose_result.rename(columns={'LTeamID':'TeamID', 'LScore':'Score'}, inplace=True)
 season_result = pd.concat(( season_win_result, season_lose_result)).reset_index(drop=True)
 season_score = season_result.groupby(['Season', 'TeamID'])['Score'].sum().reset_index()
teams = pd.read_csv(FILEDIR / 'MDataFiles_Stage1/MTeams.csv', usecols=['TeamID', 'FirstD1Season'])
 teams['FirstD1Season'] = 2020 - teams['FirstD1Season']
teams = pd.concat(
 [teams.query('TeamID in @team' ).assign(Season=season)for season, team in tourney_teams.items() ])
 teams = teams.set_index(['Season', 'TeamID'] ).sort_index()<load_from_csv>
test_df = pd.merge(test_df, tourney_seed, left_on=['Season', 'WTeamID'], right_on=['Season', 'TeamID'], how='left')
 test_df.rename(columns={'Seed':'Seed1'}, inplace=True)
 test_df = test_df.drop('TeamID', axis=1)
 test_df = pd.merge(test_df, tourney_seed, left_on=['Season', 'LTeamID'], right_on=['Season', 'TeamID'], how='left')
 test_df.rename(columns={'Seed':'Seed2'}, inplace=True)
 test_df = test_df.drop('TeamID', axis=1)
 test_df = pd.merge(test_df, season_score, left_on=['Season', 'WTeamID'], right_on=['Season', 'TeamID'], how='left')
 test_df.rename(columns={'Score':'ScoreT1'}, inplace=True)
 test_df = test_df.drop('TeamID', axis=1)
 test_df = pd.merge(test_df, season_score, left_on=['Season', 'LTeamID'], right_on=['Season', 'TeamID'], how='left')
 test_df.rename(columns={'Score':'ScoreT2'}, inplace=True)
 test_df = test_df.drop('TeamID', axis=1 )
seeds = pd.read_csv(FILEDIR / 'MDataFiles_Stage1/MNCAATourneySeeds.csv')
 seeds = pd.concat(
 [seeds.query('Season == @season and TeamID in @teams')for season, teams in tourney_teams.items() ])
 seeds = seeds.set_index(['Season', 'TeamID'] ).sort_index()
seeds['Region'] = seeds['Seed'].str[0]
seeds['Number'] = seeds['Seed'].str[1:3].astype(int)
 del seeds['Seed']<load_from_csv>
test_df['Seed1'] = test_df['Seed1'].map(lambda x: get_seed(x))
test_df['Seed2'] = test_df['Seed2'].map(lambda x: get_seed(x))
test_df['Seed_diff'] = test_df['Seed1'] - test_df['Seed2']
test_df['ScoreT_diff'] = test_df['ScoreT1'] - test_df['ScoreT2']
test_df = test_df.drop(['ID', 'Pred', 'Season', 'WTeamID', 'LTeamID'], axis=1)
 test_df
regular = pd.read_csv(FILEDIR / 'MDataFiles_Stage1/MRegularSeasonDetailedResults.csv')
 regular = regular.drop(columns=['DayNum', 'LTeamID'])
 regular = pd.concat(
 [regular.query('Season == @season and WTeamID in @teams')for season, teams in tourney_teams.items() ])
 regular = regular.groupby(['Season', 'WTeamID'] ).sum()
regular = regular.rename_axis(index=['Season', 'TeamID'] )<concatenate>
X = train_df.drop('result', axis=1)
 y = train_df.result
ctcsr = pd.concat([coaches, teams, conferences, seeds, regular], axis=1 )<load_from_csv>
params_lgb = {'num_leaves': lgb_num_leaves_max,
'min_data_in_leaf': lgb_in_leaf,
'objective': 'binary',
'max_depth': -1,
'learning_rate': lgb_lr,
"boosting_type": "gbdt",
"bagging_seed": lgb_bagging,
"metric": 'logloss',
"verbosity": -1,
'random_state': 42,
}
result = pd.read_csv(FILEDIR / 'MDataFiles_Stage1/MNCAATourneyCompactResults.csv')
 result = result[result['Season'] >= 2015].set_index(['Season', 'WTeamID', 'LTeamID'] )<concatenate>
NFOLDS = 5
folds = KFold(n_splits=NFOLDS)
 columns = X.columns
splits = folds.split(X, y)
 y_preds_lgb = np.zeros(test_df.shape[0])
 y_train_lgb = np.zeros(X.shape[0])
 y_oof = np.zeros(X.shape[0])
 feature_importances = pd.DataFrame()
feature_importances['feature'] = columns
for fold_n,(train_index, valid_index)in enumerate(splits):
print('Fold:',fold_n+1)
 X_train, X_valid = X[columns].iloc[train_index], X[columns].iloc[valid_index]
y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]
dtrain = lgb.Dataset(X_train, label=y_train)
 dvalid = lgb.Dataset(X_valid, label=y_valid)
 clf = lgb.train(params_lgb, dtrain, 10000, valid_sets = [dtrain, dvalid], verbose_eval=200)
 feature_importances[f'fold_{fold_n + 1}'] = clf.feature_importance()
y_pred_valid = clf.predict(X_valid)
 y_oof[valid_index] = y_pred_valid
y_train_lgb += clf.predict(X)/ NFOLDS
y_preds_lgb += clf.predict(test_df)/ NFOLDS
del X_train, X_valid, y_train, y_valid
gc.collect()
merged_teams = pd.concat(
 [ctcsr.loc[[(season, wteam),(season, lteam)], :] for season, wteam, lteam, in result.index])
 team1 = merged_teams.iloc[::2, :].reset_index('TeamID')
 team2 = merged_teams.iloc[1::2, :].reset_index('TeamID')
 merged_teams = pd.concat([
pd.concat([team1.add_suffix('1'), team2.add_suffix('2')], axis=1 ).assign(Res=1),
pd.concat([team2.add_suffix('1'), team1.add_suffix('2')], axis=1 ).assign(Res=0),
] ).reset_index().set_index(['Season', 'TeamID1', 'TeamID2'] ).sort_index()<normalization>
params_xgb = {'max_depth': xgb_max_depth,
'objective': 'binary:logistic',
'min_child_weight': xgb_min_child_weight,
'learning_rate': xgb_lr,
'eta' : 0.3,
'subsample': 0.8,
'eval_metric': 'logloss',
'colsample_bylevel': 1
}
x_columns = merged_teams.columns[merged_teams.columns != 'Res']
X = merged_teams[x_columns]
columns_number = X.select_dtypes(include='number' ).columns
X.loc[:, columns_number] = MinMaxScaler().fit_transform(X[columns_number])
 X = pd.get_dummies(X, columns=x_columns[X.dtypes == 'object'])
 X<prepare_x_and_y>
NFOLDS = 5
folds = KFold(n_splits=NFOLDS)
 columns = X.columns
splits = folds.split(X, y)
 y_preds_xgb = np.zeros(test_df.shape[0])
 y_train_xgb = np.zeros(X.shape[0])
 y_oof_xgb = np.zeros(X.shape[0])
 train_df_set = xgb.DMatrix(X)
 test_set = xgb.DMatrix(test_df)
 for fold_n,(train_index, valid_index)in enumerate(splits):
print('Fold:',fold_n+1)
 X_train, X_valid = X[columns].iloc[train_index], X[columns].iloc[valid_index]
y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]
train_set = xgb.DMatrix(X_train, y_train)
 val_set = xgb.DMatrix(X_valid, y_valid)
 clf = xgb.train(params_xgb, train_set, num_boost_round=xgb_num_boost_round_max, evals=[(train_set, 'train'),(val_set, 'val')], verbose_eval=100)
 y_train_xgb += clf.predict(train_df_set)/ NFOLDS
y_preds_xgb += clf.predict(test_set)/ NFOLDS
del X_train, X_valid, y_train, y_valid
gc.collect()
y = merged_teams['Res']
y<choose_model_class>
%%time
scaler = StandardScaler()
train_log = pd.DataFrame(
 scaler.fit_transform(X),
columns=X.columns,
index=X.index 
)
 test_log = pd.DataFrame(
 scaler.transform(test_df),
columns=test_df.columns,
index=test_df.index
)
clfs = {}
clfs['SVC'] = {
'instance': SVC(probability=True),
'params': [
{'kernel': ['linear'], 'C': [0.01, 0.05, 0.1, 0.5, 1]},
{'kernel': ['rbf'], 'C': [1, 10, 50, 100, 250], 'gamma': [0.1, 0.2, 0.3]}
]
}
clfs['RandomForestClassifier'] = {
'instance': RandomForestClassifier(n_jobs=-1),
'params': {
'n_estimators': [25, 50, 100],
'criterion': ['gini', 'entropy'],
'max_depth': [10, 25, 50, None]
}
}
clfs['LogisticRegression'] = {
'instance': LogisticRegression(max_iter=200, n_jobs=-1),
'params': [
{'penalty': ['l2'], 'C': [0.1, 0.5, 1, 5, 10]},
{'penalty': ['l1'], 'solver': ['liblinear', 'saga'], 'C': [0.1, 0.5, 1, 5, 10]},
{'penalty': ['elasticnet'], 'C': [0.1, 0.5, 1, 5, 10], 'l1_ratio': [0.1, 0.3, 0.5, 0.7, 0.9]}
]
}<train_on_grid>
logreg = LogisticRegression()
logreg.fit(train_log, y)
 coeff_logreg = pd.DataFrame(train_log.columns.delete(0))
coeff_logreg.columns = ['feature']
coeff_logreg["score_logreg"] = pd.Series(logreg.coef_[0])
 coeff_logreg.sort_values(by='score_logreg', ascending=False )
for clf_name, clf in clfs.items() :
print('<{}>'.format(clf_name))
print(' training...')
 gs = GridSearchCV(clf['instance'], param_grid=clf['params'], cv=5, n_jobs=-1)
 gs.fit(X, y)
 print(' best_score: {:.3f}'.format(gs.best_score_))
print(' best_params: {}'.format(gs.best_params_))
clfs[clf_name]['best_params'] = gs.best_params_<train_model>
eli5.show_weights(logreg )
for clf_name, clf in clfs.items() :
clf['best_estimator'] = clf['instance'].set_params(**clf['best_params'] ).fit(X, y )<find_best_params>
y_logreg_train = logreg.predict(train_log)
 y_logreg_pred = logreg.predict(test_log )
[clf['best_estimator'] for clf in clfs.values() ]<train_on_grid>
def plot_cm(y_true, y_pred, title, figsize=(7,6)) :
y_pred = y_pred.round().astype(int)
 cm = confusion_matrix(y_true, y_pred, labels=np.unique(y_true))
cm_sum = np.sum(cm, axis=1, keepdims=True)
 cm_perc = cm / cm_sum.astype(float)* 100
annot = np.empty_like(cm ).astype(str)
 nrows, ncols = cm.shape
for i in range(nrows):
for j in range(ncols):
c = cm[i, j]
p = cm_perc[i, j]
if i == j:
s = cm_sum[i]
annot[i, j] = '%.1f%%
%d/%d' %(p, c, s)
 elif c == 0:
annot[i, j] = ''
else:
annot[i, j] = '%.1f%%
%d' %(p, c)
 cm = pd.DataFrame(cm, index=np.unique(y_true), columns=np.unique(y_true))
cm.index.name = 'Actual'
cm.columns.name = 'Predicted'
fig, ax = plt.subplots(figsize=figsize)
 plt.title(title)
 sns.heatmap(cm, cmap= "YlGnBu", annot=annot, fmt='', ax=ax )
vote = VotingClassifier(
 estimators=[(clf_name, clf['best_estimator'])for clf_name, clf in clfs.items() ],
voting='soft',
n_jobs=-1 
)
 vote.fit(X, y)
 clfs['Vote'] = {}
clfs['Vote']['best_estimator'] = vote<compute_test_metric>
y_preds = w_lgb*y_preds_lgb + w_xgb*y_preds_xgb + w_logreg*y_logreg_pred
for clf_name, clf in clfs.items() :
score = accuracy_score(y, clf['best_estimator'].predict(X))
print(clf_name, score )<predict_on_test>
sub['Pred'] = y_preds
sub.head()
predict = pd.DataFrame(
 {
**{'pp_' + clf_name: clf['best_estimator'].predict_proba(X)[:, 1] for clf_name, clf in clfs.items() },
**{'p_' + clf_name: clf['best_estimator'].predict(X)for clf_name, clf in clfs.items() }
},
index=X.index)
 predict<feature_engineering>
sub.to_csv('submission.csv', index=False )
for column in predict.columns:
sub[column] = 0.5
mask = [idx for idx in sub.index if idx in X.index]
sub.loc[mask, predict.columns] = predict.loc[mask, predict.columns]
sub<save_to_csv>
sub.to_csv('submission.csv', index=False )