Kaggle competition: Google Cloud & NCAA® ML Competition 2020-NCAAM
<SOS> metric: logloss Kaggle data source: google-cloud-ncaa-march-madness-2020-division-1-mens-tournament<count_values>
import pandas as pd
import numpy as np
from sklearn.linear_model import LogisticRegression
import matplotlib.pyplot as plt
from sklearn.utils import shuffle
from sklearn.model_selection import GridSearchCV
all_data.dtypes.value_counts()<categorify>
tourney_result = pd.read_csv('.. /input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MDataFiles_Stage1/MNCAATourneyCompactResults.csv')
 tourney_seed = pd.read_csv('.. /input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MDataFiles_Stage1/MNCAATourneySeeds.csv' )
all_data = pd.get_dummies(all_data,columns=cal_list)
 all_data.shape<import_modules>
tourney_result = tourney_result.drop(['DayNum', 'WScore', 'LScore', 'WLoc', 'NumOT'], axis=1)
 tourney_result
from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor
from sklearn.model_selection import KFold, cross_val_score, train_test_split
from sklearn.metrics import mean_squared_error
import xgboost as xgb
from sklearn.model_selection import GridSearchCV
import lightgbm as lgb<split>
tourney_result = pd.merge(tourney_result, tourney_seed, left_on=['Season', 'WTeamID'], right_on=['Season', 'TeamID'], how='left')
 tourney_result.rename(columns={'Seed':'WSeed'}, inplace=True)
 tourney_result = tourney_result.drop('TeamID', axis=1)
 tourney_result = pd.merge(tourney_result, tourney_seed, left_on=['Season', 'LTeamID'], right_on=['Season', 'TeamID'], how='left')
 tourney_result.rename(columns={'Seed':'LSeed'}, inplace=True)
 tourney_result = tourney_result.drop('TeamID', axis=1)
 tourney_result
train_x, valid_x, train_y, valid_y = train_test_split(
 train_x,
train_y,
test_size=0.3,
random_state=0 )<train_model>
def get_seed(x):
return int(x[1:3])
 tourney_result['WSeed'] = tourney_result['WSeed'].map(lambda x: get_seed(x))
tourney_result['LSeed'] = tourney_result['LSeed'].map(lambda x: get_seed(x))
tourney_result
dtrain = xgb.DMatrix(train_x, label=train_y)
 dvalid = xgb.DMatrix(valid_x,label=valid_y)
 num_round = 5000
evallist = [(dvalid, 'eval'),(dtrain, 'train')]
evals_result = {}
param = {
'max_depth': 3,
'eta': 0.01,
'objective': 'reg:squarederror',
}
bst = xgb.train(
 param, dtrain,
num_round,
evallist,
evals_result=evals_result,
early_stopping_rounds = 1000 )<save_to_csv>
season_result = pd.read_csv('.. /input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MDataFiles_Stage1/MRegularSeasonCompactResults.csv' )
dtest = xgb.DMatrix(test_x)
 my_submission = pd.DataFrame()
my_submission["Id"] = test_ID
my_submission["SalePrice"] = np.exp(bst.predict(dtest))
my_submission.to_csv('submission.csv', index=False )<load_from_csv>
season_win_result = season_result[['Season', 'WTeamID', 'WScore']]
season_lose_result = season_result[['Season', 'LTeamID', 'LScore']]
season_win_result.rename(columns={'WTeamID':'TeamID', 'WScore':'Score'}, inplace=True)
 season_lose_result.rename(columns={'LTeamID':'TeamID', 'LScore':'Score'}, inplace=True)
 season_result = pd.concat(( season_win_result, season_lose_result)).reset_index(drop=True)
 season_result
train_data = pd.read_csv("/kaggle/input/titanic/train.csv")
 test_data = pd.read_csv("/kaggle/input/titanic/test.csv" )<feature_engineering>
season_score = season_result.groupby(['Season', 'TeamID'])['Score'].sum().reset_index()
season_score
frames = [train_data,test_data]
full_data = pd.concat(frames)
 master_data = full_data
print(train_data.shape)
 print(test_data.shape)
 print(full_data.shape)
 full_data = full_data.drop(['PassengerId','Survived','Ticket','Cabin'],axis = 1)
 print(full_data.shape)
 full_data['Name'] = full_data['Name'].str.rsplit(',' ).str[-1]
full_data['Name'] = full_data['Name'].str.split('.' ).str[0]
full_data['Name'] = full_data['Name'].replace(['Lady','the Countess','Capt', 'Col',
'Dona','Don', 'Dr', 'Major','','Rev', 'Sir', 'Jonkheer'], 'Rare',regex=True)
 full_data['Name'] = full_data['Name'].replace('Mlle', 'Miss', regex=True)
 full_data['Name'] = full_data['Name'].replace('Ms', 'Miss',regex=True)
 full_data['Name'] = full_data['Name'].replace('Mme', 'Mrs',regex=True)
 pd.crosstab(full_data['Name'], full_data['Sex'])
 full_data.dropna(subset = ["Age"], inplace=True)
 print(np.where(pd.isnull(full_data)))
 full_data.dropna(how='any', inplace=True)
 print(np.where(pd.isnull(full_data)))
 y = full_data['Age']
X = full_data.drop(['Age'],axis = 1)
 X_train ,X_test ,y_train ,y_test = train_test_split(X ,y ,test_size=0.2,random_state = 14)
 numerical_features = ['SibSp','Parch','Fare']
categorical_features = ['Sex','Embarked','Pclass','Name']
numerical_pipeline = make_pipeline(MinMaxScaler())
 categorical_pipeline = make_pipeline(OneHotEncoder())
 preprocessor = make_column_transformer(( numerical_pipeline, numerical_features),(categorical_pipeline, categorical_features))
model = make_pipeline(preprocessor, GradientBoostingRegressor())
 model.fit(X_train, y_train)
 params = {'gradientboostingregressor__n_estimators' : [2000]
, 'gradientboostingregressor__learning_rate' : [0.01]}
grid = GridSearchCV(model, param_grid= params, cv=5)
 grid.fit(X_train, y_train)
 print(grid.best_params_)
 print('train score :' , grid.best_score_)
 best_model = grid.best_estimator_
print('test score :', best_model.score(X_test,y_test))
train_data.dropna(subset = ["Fare","Embarked"], inplace=True)
 train_data = train_data.drop(['Ticket','Cabin'], axis = 1)
 print(train_data.shape)
 train_data['Name'] = train_data['Name'].str.rsplit(',' ).str[-1]
train_data['Name'] = train_data['Name'].str.split('.' ).str[0]
train_data['Name'] = train_data['Name'].replace(['Lady','the Countess','Capt','Col','Dona','Don', 'Dr','Major','','Rev','Sir','Jonkheer'], 'Rare',regex=True)
 train_data['Name'] = train_data['Name'].replace('Mlle', 'Miss', regex=True)
 train_data['Name'] = train_data['Name'].replace('Ms', 'Miss',regex=True)
 train_data['Name'] = train_data['Name'].replace('Mme', 'Mrs',regex=True)
 train_data
train_data['age_predicted'] = best_model.predict(train_data)
 train_data['Age'].fillna(train_data['age_predicted'], inplace=True)
 train_data = train_data.drop(['age_predicted'],axis = 1)
 train_data
train_data['family_size'] = train_data['SibSp'] + train_data['Parch']+1
train_data
split = [0, 3, 14, 24, 34, 44, 54, 64]
names = ['0', '1', '2', '3', '4','5','6','7']
d = dict(enumerate(names, 1))
train_data['Age'] = np.vectorize(d.get )(np.digitize(train_data['Age'], split))
train_data
master_y = train_data['Survived']
master_X = train_data.drop(['Survived'], axis = 1)
 print('master_y',master_y.shape)
 print('master_X',master_X.shape)
 master_X_train ,master_X_test ,master_y_train ,master_y_test = train_test_split(master_X ,master_y ,test_size=0.2,random_state = 14)
 master_numerical_features = ['SibSp','Parch','Fare']
master_categorical_features = ['Sex','Embarked','Pclass','Name','family_size']
master_numerical_pipeline = make_pipeline(MinMaxScaler())
 master_categorical_pipeline = make_pipeline(OneHotEncoder())
 master_preprocessor = make_column_transformer(( master_numerical_pipeline, master_numerical_features),(master_categorical_pipeline, master_categorical_features))
master_model = make_pipeline(master_preprocessor, GradientBoostingClassifier())
 master_model.fit(master_X_train, master_y_train)
 master_model.fit(master_X_test, master_y_test)
 master_params = {'gradientboostingclassifier__n_estimators' : [1000]
, 'gradientboostingclassifier__learning_rate' : [0.01]}
master_grid = GridSearchCV(master_model, param_grid= master_params, cv=5)
 master_grid.fit(master_X_train, master_y_train)
 print(master_grid.best_params_)
 print('Master train score :' , master_grid.best_score_)
 master_best_model = master_grid.best_estimator_
print('Master test score :', master_best_model.score(master_X_test,master_y_test))
test_data = test_data.drop(['Ticket','Cabin'], axis = 1)
 test_data['Fare'].fillna(value = test_data['Fare'].median() , inplace = True)
 test_data['Name'] = test_data['Name'].str.rsplit(',' ).str[-1]
test_data['Name'] = test_data['Name'].str.split('.' ).str[0]
test_data['Name'] = test_data['Name'].replace(['Lady','the Countess','Capt','Col','Dona','Don', 'Dr','Major','','Rev','Sir','Jonkheer'], 'Rare',regex=True)
 test_data['Name'] = test_data['Name'].replace('Mlle', 'Miss', regex=True)
 test_data['Name'] = test_data['Name'].replace('Ms', 'Miss',regex=True)
 test_data['Name'] = test_data['Name'].replace('Mme', 'Mrs',regex=True)
 test_data['age_predicted'] = best_model.predict(test_data)
 test_data['Age'].fillna(test_data['age_predicted'], inplace=True)
 test_data = test_data.drop(['age_predicted'],axis = 1)
 split = [0, 3, 14, 24, 34, 44, 54, 64]
names = ['0', '1', '2', '3', '4','5','6','7']
d = dict(enumerate(names, 1))
test_data['Age'] = np.vectorize(d.get )(np.digitize(test_data['Age'], split))
test_data
test_data['family_size'] = test_data['SibSp'] + test_data['Parch']+1
test_data
predictions = master_best_model.predict(test_data ).astype(int)
 predictions
output = pd.DataFrame({'PassengerId': test_data.PassengerId, 'Survived': predictions})
 output.to_csv('my_submission.csv', index=False)
 print("Your submission was successfully saved!")
<load_from_csv>
tourney_result = pd.merge(tourney_result, season_score, left_on=['Season', 'WTeamID'], right_on=['Season', 'TeamID'], how='left')
 tourney_result.rename(columns={'Score':'WScoreT'}, inplace=True)
 tourney_result = tourney_result.drop('TeamID', axis=1)
 tourney_result = pd.merge(tourney_result, season_score, left_on=['Season', 'LTeamID'], right_on=['Season', 'TeamID'], how='left')
 tourney_result.rename(columns={'Score':'LScoreT'}, inplace=True)
 tourney_result = tourney_result.drop('TeamID', axis=1)
 tourney_result
train = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/train.csv")
 test = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/test.csv")
 train.columns<prepare_x_and_y>
tourney_win_result = tourney_result.drop(['Season', 'WTeamID', 'LTeamID'], axis=1)
 tourney_win_result.rename(columns={'WSeed':'Seed1', 'LSeed':'Seed2', 'WScoreT':'ScoreT1', 'LScoreT':'ScoreT2'}, inplace=True)
 tourney_win_result
train_x = train.drop("SalePrice",axis=1)
 train_y = train["SalePrice"]
all_data = pd.concat([train_x,test],axis=0,sort=True)
 train_ID = train['Id']
test_ID = test['Id']
all_data.drop("Id", axis = 1, inplace = True)
 print("train_x: "+str(train_x.shape))
print("train_y: "+str(train_y.shape))
print("test: "+str(test.shape))
print("all_data: "+str(all_data.shape))<sort_values>
tourney_lose_result = tourney_win_result.copy()
tourney_lose_result['Seed1'] = tourney_win_result['Seed2']
tourney_lose_result['Seed2'] = tourney_win_result['Seed1']
tourney_lose_result['ScoreT1'] = tourney_win_result['ScoreT2']
tourney_lose_result['ScoreT2'] = tourney_win_result['ScoreT1']
tourney_lose_result
all_data_na = all_data.isnull().sum() [all_data.isnull().sum() >0].sort_values(ascending=False)
 all_data_na<sort_values>
tourney_win_result['Seed_diff'] = tourney_win_result['Seed1'] - tourney_win_result['Seed2']
tourney_win_result['ScoreT_diff'] = tourney_win_result['ScoreT1'] - tourney_win_result['ScoreT2']
tourney_lose_result['Seed_diff'] = tourney_lose_result['Seed1'] - tourney_lose_result['Seed2']
tourney_lose_result['ScoreT_diff'] = tourney_lose_result['ScoreT1'] - tourney_lose_result['ScoreT2']
na_col_list = all_data.isnull().sum() [all_data.isnull().sum() >0].index.tolist()
all_data[na_col_list].dtypes.sort_values()<data_type_conversions>
tourney_win_result['result'] = 1
tourney_lose_result['result'] = 0
tourney_result = pd.concat(( tourney_win_result, tourney_lose_result)).reset_index(drop=True)
 tourney_result
all_data['LotFrontage'] = all_data.groupby('Neighborhood')['LotFrontage'].transform(lambda x: x.fillna(x.median()))
float_list = all_data[na_col_list].dtypes[all_data[na_col_list].dtypes == "float64"].index.tolist()
obj_list = all_data[na_col_list].dtypes[all_data[na_col_list].dtypes == "object"].index.tolist()
all_data[float_list] = all_data[float_list].fillna(0)
 all_data[obj_list] = all_data[obj_list].fillna("None")
 all_data.isnull().sum() [all_data.isnull().sum() > 0]<data_type_conversions>
test_df = pd.read_csv('.. /input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MSampleSubmissionStage1_2020.csv' )
all_data['MSSubClass'] = all_data['MSSubClass'].apply(str)
 all_data['YrSold'] = all_data['YrSold'].astype(str)
 all_data['MoSold'] = all_data['MoSold'].astype(str )<filter>
test_df['Season'] = test_df['ID'].map(lambda x: int(x[:4]))
test_df['WTeamID'] = test_df['ID'].map(lambda x: int(x[5:9]))
test_df['LTeamID'] = test_df['ID'].map(lambda x: int(x[10:14]))
test_df
skewed_feats_over = skewed_feats[abs(skewed_feats)> 0.5].index
for i in skewed_feats_over:
print(min(all_data[i]))<feature_engineering>
test_df = pd.merge(test_df, tourney_seed, left_on=['Season', 'WTeamID'], right_on=['Season', 'TeamID'], how='left')
 test_df.rename(columns={'Seed':'Seed1'}, inplace=True)
 test_df = test_df.drop('TeamID', axis=1)
 test_df = pd.merge(test_df, tourney_seed, left_on=['Season', 'LTeamID'], right_on=['Season', 'TeamID'], how='left')
 test_df.rename(columns={'Seed':'Seed2'}, inplace=True)
 test_df = test_df.drop('TeamID', axis=1)
 test_df = pd.merge(test_df, season_score, left_on=['Season', 'WTeamID'], right_on=['Season', 'TeamID'], how='left')
 test_df.rename(columns={'Score':'ScoreT1'}, inplace=True)
 test_df = test_df.drop('TeamID', axis=1)
 test_df = pd.merge(test_df, season_score, left_on=['Season', 'LTeamID'], right_on=['Season', 'TeamID'], how='left')
 test_df.rename(columns={'Score':'ScoreT2'}, inplace=True)
 test_df = test_df.drop('TeamID', axis=1)
 test_df
all_data["FeetPerRoom"] = all_data["TotalSF"]/all_data["TotRmsAbvGrd"]
all_data['YearBuiltAndRemod']=all_data['YearBuilt']+all_data['YearRemodAdd']
all_data['Total_Bathrooms'] =(all_data['FullBath'] +(0.5 * all_data['HalfBath'])+
all_data['BsmtFullBath'] +(0.5 * all_data['BsmtHalfBath']))
all_data['Total_porch_sf'] =(all_data['OpenPorchSF'] + all_data['3SsnPorch'] +
all_data['EnclosedPorch'] + all_data['ScreenPorch'] +
all_data['WoodDeckSF'])
 all_data['haspool'] = all_data['PoolArea'].apply(lambda x: 1 if x > 0 else 0)
 all_data['has2ndfloor'] = all_data['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)
 all_data['hasgarage'] = all_data['GarageArea'].apply(lambda x: 1 if x > 0 else 0)
 all_data['hasbsmt'] = all_data['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 all_data['hasfireplace'] = all_data['Fireplaces'].apply(lambda x: 1 if x > 0 else 0 )<count_values>
test_df['Seed1'] = test_df['Seed1'].map(lambda x: get_seed(x))
test_df['Seed2'] = test_df['Seed2'].map(lambda x: get_seed(x))
test_df['Seed_diff'] = test_df['Seed1'] - test_df['Seed2']
test_df['ScoreT_diff'] = test_df['ScoreT1'] - test_df['ScoreT2']
test_df = test_df.drop(['ID', 'Pred', 'Season', 'WTeamID', 'LTeamID'], axis=1)
 test_df
all_data.dtypes.value_counts()<categorify>
X = tourney_result.drop('result', axis=1)
 y = tourney_result.result
all_data = pd.get_dummies(all_data,columns=cal_list)
 all_data.shape<import_modules>
params = {'num_leaves': 470,
'min_child_weight': 0.034,
'feature_fraction': 0.379,
'bagging_fraction': 0.418,
'min_data_in_leaf': 106,
'objective': 'binary',
'max_depth': -1,
'learning_rate': 0.01,
"boosting_type": "gbdt",
"bagging_seed": 11,
"metric": 'logloss',
"verbosity": -1,
'reg_alpha': 0.3899,
'reg_lambda': 0.648,
'random_state': 47,
}
from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor
from sklearn.model_selection import KFold, cross_val_score, train_test_split
from sklearn.metrics import mean_squared_error
import xgboost as xgb
from sklearn.model_selection import GridSearchCV
import lightgbm as lgb<split>
NFOLDS = 5
folds = KFold(n_splits=NFOLDS)
 columns = X.columns
splits = folds.split(X, y)
 y_preds = np.zeros(test_df.shape[0])
 y_oof = np.zeros(X.shape[0])
 feature_importances = pd.DataFrame()
feature_importances['feature'] = columns
for fold_n,(train_index, valid_index)in enumerate(splits):
print('Fold:',fold_n+1)
 X_train, X_valid = X[columns].iloc[train_index], X[columns].iloc[valid_index]
y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]
dtrain = lgb.Dataset(X_train, label=y_train)
 dvalid = lgb.Dataset(X_valid, label=y_valid)
 clf = lgb.train(params, dtrain, 10000, valid_sets = [dtrain, dvalid], verbose_eval=200)
 feature_importances[f'fold_{fold_n + 1}'] = clf.feature_importance()
y_pred_valid = clf.predict(X_valid)
 y_oof[valid_index] = y_pred_valid
y_preds += clf.predict(test_df)/ NFOLDS
del X_train, X_valid, y_train, y_valid
gc.collect()
train_x, valid_x, train_y, valid_y = train_test_split(
 train_x,
train_y,
test_size=0.3,
random_state=0)
<train_model>
submission_df = pd.read_csv('.. /input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MSampleSubmissionStage1_2020.csv')
 submission_df['Pred'] = y_preds
submission_df
dtrain = xgb.DMatrix(train_x, label=train_y)
 dvalid = xgb.DMatrix(valid_x,label=valid_y)
 num_round = 5000
evallist = [(dvalid, 'eval'),(dtrain, 'train')]
evals_result = {}
param = {
'max_depth': 3,
'eta': 0.01,
'objective': 'reg:squarederror',
}
bst = xgb.train(
 param, dtrain,
num_round,
evallist,
evals_result=evals_result,
early_stopping_rounds=1000 )<save_to_csv>
submission_df.to_csv('submission.csv', index=False )
<set_options><EOS> 
feature_importances['average'] = feature_importances[[f'fold_{fold_n + 1}' for fold_n in range(folds.n_splits)]].mean(axis=1)
 feature_importances.to_csv('feature_importances.csv')
 plt.figure(figsize=(10, 6))
sns.barplot(data=feature_importances.sort_values(by='average', ascending=False ).head(7), x='average', y='feature');
plt.title('7 TOP feature importance over {} folds average'.format(folds.n_splits)) ;