Kaggle competition: Google Cloud & NCAA® ML Competition 2020-NCAAW
learn.fit_one_cycle(5, slice(1.58E-06, lr/5))<save_model>
import pandas as pd
import numpy as np
from sklearn.linear_model import LogisticRegression
import matplotlib.pyplot as plt
from sklearn.utils import shuffle
from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import KFold
import lightgbm as lgb
import xgboost as xgb
from xgboost import XGBClassifier
import gc
import matplotlib.pyplot as plt
from sklearn import preprocessing
import pymc3 as pm
import matplotlib.pyplot as plt
import seaborn as sns
learn.save('stage-2-256-rn50' )<train_model>
Tourney_Compact_Results = pd.read_csv('.. /input/google-cloud-ncaa-march-madness-2020-division-1-womens-tournament/WDataFiles_Stage1/WNCAATourneyCompactResults.csv')
 Tourney_Seeds = pd.read_csv('.. /input/google-cloud-ncaa-march-madness-2020-division-1-womens-tournament/WDataFiles_Stage1/WNCAATourneySeeds.csv' )
learn.fit_one_cycle(5, slice(1.58E-06, lr/5))<save_model>
RegularSeason_Compact_Results = pd.read_csv('.. /input/google-cloud-ncaa-march-madness-2020-division-1-womens-tournament/WDataFiles_Stage1/WRegularSeasonCompactResults.csv')
 MSeasons = pd.read_csv('.. /input/google-cloud-ncaa-march-madness-2020-division-1-womens-tournament/WDataFiles_Stage1/WSeasons.csv')
 MTeams=pd.read_csv('.. /input/google-cloud-ncaa-march-madness-2020-division-1-womens-tournament/WDataFiles_Stage1/WTeams.csv' )
learn.save('stage-2-256-rn50-10e' )<define_variables>
Tourney_Results_Compact=pd.merge(Tourney_Compact_Results, Tourney_Seeds, left_on=['Season', 'WTeamID'], right_on=['Season', 'TeamID'], how='left')
 Tourney_Results_Compact.rename(columns={'Seed':'WinningSeed'},inplace=True)
 Tourney_Results_Compact=Tourney_Results_Compact.drop(['TeamID'],axis=1)
 Tourney_Results_Compact = pd.merge(Tourney_Results_Compact, Tourney_Seeds, left_on=['Season', 'LTeamID'], right_on=['Season', 'TeamID'], how='left')
 Tourney_Results_Compact.rename(columns={'Seed':'LoosingSeed'}, inplace=True)
 Tourney_Results_Compact=Tourney_Results_Compact.drop(['TeamID','NumOT','WLoc'],axis=1)
 Tourney_Results_Compact
test = ImageList.from_folder(spect_dir/'test/')
 len(test )<predict_on_test>
Tourney_Results_Compact=Tourney_Results_Compact.drop(['WScore','LScore'],axis=1)
 Tourney_Results_Compact.head()
predictor = load_learner('.', test=test, num_workers=0)
 preds, _ = predictor.get_preds(ds_type=DatasetType.Test)
 fpreds = preds[:, 1:].reshape(-1, )<define_variables>
Tourney_Results_Compact['WinningSeed'] = Tourney_Results_Compact['WinningSeed'].str.extract('(\d+)', expand=True)
 Tourney_Results_Compact['LoosingSeed'] = Tourney_Results_Compact['LoosingSeed'].str.extract('(\d+)', expand=True)
 Tourney_Results_Compact.WinningSeed = pd.to_numeric(Tourney_Results_Compact.WinningSeed, errors='coerce')
 Tourney_Results_Compact.LoosingSeed = pd.to_numeric(Tourney_Results_Compact.LoosingSeed, errors='coerce' )
names = [f.stem for f in predictor.data.test_ds.items]
fnames = [x + '.wav_classnumber_' + str(i)for x in names for i in range(1, len(data.classes)) ]<save_to_csv>
season_winning_team = RegularSeason_Compact_Results[['Season', 'WTeamID', 'WScore']]
season_losing_team = RegularSeason_Compact_Results[['Season', 'LTeamID', 'LScore']]
season_winning_team.rename(columns={'WTeamID':'TeamID','WScore':'Score'}, inplace=True)
 season_losing_team.rename(columns={'LTeamID':'TeamID','LScore':'Score'}, inplace=True)
 RegularSeason_Compact_Results = pd.concat(( season_winning_team, season_losing_team)).reset_index(drop=True)
 RegularSeason_Compact_Results
test_df = pd.DataFrame({'ID':fnames, 'Probability':fpreds}, columns=['ID', 'Probability'])
 test_df.to_csv('submission.csv', index=False )<import_modules>
RegularSeason_Compact_Results_Final = RegularSeason_Compact_Results.groupby(['Season', 'TeamID'])['Score'].sum().reset_index()
RegularSeason_Compact_Results_Final
import os
import sys
import operator
import numpy as np
import pandas as pd
from scipy import sparse
import xgboost as xgb
from sklearn import model_selection, preprocessing, ensemble
from sklearn.metrics import log_loss
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer 
<train_model>
Tourney_Results_Compact = pd.merge(Tourney_Results_Compact, RegularSeason_Compact_Results_Final, left_on=['Season', 'WTeamID'], right_on=['Season', 'TeamID'], how='left')
 Tourney_Results_Compact.rename(columns={'Score':'WScoreTotal'}, inplace=True)
 Tourney_Results_Compact
def runXGB(train_X, train_y, test_X, test_y=None,max_depth = 6, feature_names=None, seed_val=0, num_rounds=1000):
param = {}
param['objective'] = 'multi:softprob'
param['eta'] = 0.15
param['max_depth'] = max_depth
param['silent'] = 1
param['num_class'] = 3
param['eval_metric'] = "mlogloss"
param['min_child_weight'] = 1
param['subsample'] = 0.7
param['colsample_bytree'] = 0.7
param['seed'] = seed_val
num_rounds = num_rounds
plst = list(param.items())
 xgtrain = xgb.DMatrix(train_X, label=train_y)
 if test_y is not None:
xgtest = xgb.DMatrix(test_X, label=test_y)
 watchlist = [(xgtrain,'train'),(xgtest, 'test')]
model = xgb.train(plst, xgtrain, num_rounds, watchlist, early_stopping_rounds=20)
 else:
xgtest = xgb.DMatrix(test_X)
 model = xgb.train(plst, xgtrain, num_rounds)
 pred_test_y = model.predict(xgtest)
 return pred_test_y, model 
<load_from_disk>
Tourney_Results_Compact = Tourney_Results_Compact.drop('TeamID', axis=1)
 Tourney_Results_Compact = pd.merge(Tourney_Results_Compact, RegularSeason_Compact_Results_Final, left_on=['Season', 'LTeamID'], right_on=['Season', 'TeamID'], how='left')
 Tourney_Results_Compact.rename(columns={'Score':'LScoreTotal'}, inplace=True)
 Tourney_Results_Compact = Tourney_Results_Compact.drop('TeamID', axis=1)
 Tourney_Results_Compact=Tourney_Results_Compact[Tourney_Results_Compact['Season'] < 2016]
Tourney_Results_Compact
data_path = ".. /input/"
train_file = data_path + "train.json"
test_file = data_path + "test.json"
train_df = pd.read_json(train_file)
 test_df = pd.read_json(test_file)
 print(train_df.shape)
 print(test_df.shape)
<define_variables>
Tourney_Win_Results=Tourney_Results_Compact.drop(['Season','WTeamID','LTeamID','DayNum'],axis=1)
 Tourney_Win_Results
features_to_use = ["bathrooms", "bedrooms", "latitude", "longitude", "price"] 
<feature_engineering>
Tourney_Win_Results.rename(columns={'WinningSeed':'Seed1', 'LoosingSeed':'Seed2', 'WScoreTotal':'ScoreT1', 'LScoreTotal':'ScoreT2'}, inplace=True )
train_df["num_photos"] = train_df["photos"].apply(len)
 test_df["num_photos"] = test_df["photos"].apply(len)
 train_df["num_features"] = train_df["features"].apply(len)
 test_df["num_features"] = test_df["features"].apply(len)
 train_df["num_description_words"] = train_df["description"].apply(lambda x: len(x.split(" ")))
 test_df["num_description_words"] = test_df["description"].apply(lambda x: len(x.split(" ")))
 train_df["created"] = pd.to_datetime(train_df["created"])
 test_df["created"] = pd.to_datetime(test_df["created"])
 train_df["created_year"] = train_df["created"].dt.year
test_df["created_year"] = test_df["created"].dt.year
train_df["created_month"] = train_df["created"].dt.month
test_df["created_month"] = test_df["created"].dt.month
train_df["created_day"] = train_df["created"].dt.day
test_df["created_day"] = test_df["created"].dt.day
train_df["created_hour"] = train_df["created"].dt.hour
test_df["created_hour"] = test_df["created"].dt.hour
features_to_use.extend(["num_photos", "num_features", "num_description_words","created_year", "created_month", "created_day", "listing_id", "created_hour"])
<categorify>
tourney_lose_result = Tourney_Win_Results.copy()
tourney_lose_result['Seed1'] = Tourney_Win_Results['Seed2']
tourney_lose_result['Seed2'] = Tourney_Win_Results['Seed1']
tourney_lose_result['ScoreT1'] = Tourney_Win_Results['ScoreT2']
tourney_lose_result['ScoreT2'] = Tourney_Win_Results['ScoreT1']
tourney_lose_result
categorical = ["display_address", "manager_id", "building_id", "street_address"]
for f in categorical:
if train_df[f].dtype=='object':
lbl = preprocessing.LabelEncoder()
lbl.fit(list(train_df[f].values)+ list(test_df[f].values))
train_df[f] = lbl.transform(list(train_df[f].values))
test_df[f] = lbl.transform(list(test_df[f].values))
features_to_use.append(f)
<feature_engineering>
Tourney_Win_Results['Seed_diff'] = Tourney_Win_Results['Seed1'] - Tourney_Win_Results['Seed2']
Tourney_Win_Results['ScoreT_diff'] = Tourney_Win_Results['ScoreT1'] - Tourney_Win_Results['ScoreT2']
tourney_lose_result['Seed_diff'] = tourney_lose_result['Seed1'] - tourney_lose_result['Seed2']
tourney_lose_result['ScoreT_diff'] = tourney_lose_result['ScoreT1'] - tourney_lose_result['ScoreT2']
train_df['features'] = train_df["features"].apply(lambda x: " ".join(["_".join(i.split(" ")) for i in x]))
test_df['features'] = test_df["features"].apply(lambda x: " ".join(["_".join(i.split(" ")) for i in x]))
print(train_df["features"].head())
 tfidf = CountVectorizer(stop_words='english', max_features=200)
 tr_sparse = tfidf.fit_transform(train_df["features"])
 te_sparse = tfidf.transform(test_df["features"])
<prepare_x_and_y>
Tourney_Win_Results['result'] = 1
tourney_lose_result['result'] = 0
tourney_result_Final = pd.concat(( Tourney_Win_Results, tourney_lose_result)).reset_index(drop=True)
 tourney_result_Final.to_csv('Tourneyvalidate.csv', index=False )
train_X = sparse.hstack([train_df[features_to_use], tr_sparse] ).tocsr()
test_X = sparse.hstack([test_df[features_to_use], te_sparse] ).tocsr()
target_num_map = {'high':0, 'medium':1, 'low':2}
train_y = np.array(train_df['interest_level'].apply(lambda x: target_num_map[x]))
print(train_X.shape, test_X.shape)
<compute_train_metric>
tourney_result_Final1 = tourney_result_Final[[
'Seed1', 'Seed2', 'ScoreT1', 'ScoreT2', 'Seed_diff', 'ScoreT_diff', 'result']]
cv_scores = []
kf = model_selection.KFold(n_splits=5, shuffle=True, random_state=2016)
 for max_depth in [7,8]:
for dev_index, val_index in kf.split(range(train_X.shape[0])) :
dev_X, val_X = train_X[dev_index,:], train_X[val_index,:]
dev_y, val_y = train_y[dev_index], train_y[val_index]
preds, model = runXGB(dev_X, dev_y, val_X, val_y,max_depth)
 cv_scores.append(log_loss(val_y, preds))
print(cv_scores)
 break 
<save_to_csv>
tourney_result_Final1.loc[lambda x:(x['Seed1'].isin([14,15,16])) &(x['Seed2'].isin([1,2,3])) ,'result'
] = 0
<set_options>
test_df = pd.read_csv('.. /input/google-cloud-ncaa-march-madness-2020-division-1-womens-tournament/WSampleSubmissionStage1_2020.csv' )
warnings.simplefilter(action='ignore', category=FutureWarning)
 gc.enable()<load_from_csv>
test_df['Season'] = test_df['ID'].map(lambda x: int(x[:4]))
test_df['WTeamID'] = test_df['ID'].map(lambda x: int(x[5:9]))
test_df['LTeamID'] = test_df['ID'].map(lambda x: int(x[10:14]))
test_df
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )<load_from_csv>
test_df = pd.merge(test_df, Tourney_Seeds, left_on=['Season', 'WTeamID'], right_on=['Season', 'TeamID'], how='left')
 test_df.rename(columns={'Seed':'Seed1'}, inplace=True)
 test_df = test_df.drop('TeamID', axis=1)
 test_df = pd.merge(test_df, Tourney_Seeds, left_on=['Season', 'LTeamID'], right_on=['Season', 'TeamID'], how='left')
 test_df.rename(columns={'Seed':'Seed2'}, inplace=True)
 test_df = test_df.drop('TeamID', axis=1 )
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )<data_type_conversions>
test_df = pd.merge(test_df, RegularSeason_Compact_Results_Final, left_on=['Season', 'WTeamID'], right_on=['Season', 'TeamID'], how='left')
 test_df.rename(columns={'Score':'ScoreT1'}, inplace=True)
 test_df = test_df.drop('TeamID', axis=1)
 test_df = pd.merge(test_df, RegularSeason_Compact_Results_Final, left_on=['Season', 'LTeamID'], right_on=['Season', 'TeamID'], how='left')
 test_df.rename(columns={'Score':'ScoreT2'}, inplace=True)
 test_df = test_df.drop('TeamID', axis=1)
 test_df
test_df.to_csv('test_df_Test.csv', index=False )
def pre_pro(df):
df = df.astype('float32')
 col = df.columns
for i in range(len(col)) :
m = df.loc[df[col[i]] != -np.inf, col[i]].min()
df[col[i]].replace(-np.inf,m,inplace=True)
 M = df.loc[df[col[i]] != np.inf, col[i]].max()
df[col[i]].replace(np.inf,M,inplace=True)
 df.fillna(0, inplace = True)
 return df<feature_engineering>
test_df['Seed1'] = test_df['Seed1'].str.extract('(\d+)', expand=True)
 test_df['Seed2'] = test_df['Seed2'].str.extract('(\d+)', expand=True)
 test_df.Seed1 = pd.to_numeric(test_df.Seed1, errors='coerce')
 test_df.Seed2 = pd.to_numeric(test_df.Seed2, errors='coerce' )
def feat_eng(df):
df.replace(0, 0.001)
 df['follower_diff'] =(df['A_follower_count'] > df['B_follower_count'])
 df['following_diff'] =(df['A_following_count'] > df['B_following_count'])
 df['listed_diff'] =(df['A_listed_count'] > df['B_listed_count'])
 df['ment_rec_diff'] =(df['A_mentions_received'] > df['B_mentions_received'])
 df['rt_rec_diff'] =(df['A_retweets_received'] > df['B_retweets_received'])
 df['ment_sent_diff'] =(df['A_mentions_sent'] > df['B_mentions_sent'])
 df['rt_sent_diff'] =(df['A_retweets_sent'] > df['B_retweets_sent'])
 df['posts_diff'] =(df['A_posts'] > df['B_posts'])
 df['A_pop_ratio'] = df['A_mentions_sent']/df['A_listed_count']
df['A_foll_ratio'] = df['A_follower_count']/df['A_following_count']
df['A_ment_ratio'] = df['A_mentions_sent']/df['A_mentions_received']
df['A_rt_ratio'] = df['A_retweets_sent']/df['A_retweets_received']
df['B_pop_ratio'] = df['B_mentions_sent']/df['B_listed_count']
df['B_foll_ratio'] = df['B_follower_count']/df['B_following_count']
df['B_ment_ratio'] = df['B_mentions_sent']/df['B_mentions_received']
df['B_rt_ratio'] = df['B_retweets_sent']/df['B_retweets_received']
df['A/B_foll_ratio'] =(df['A_foll_ratio'] > df['B_foll_ratio'])
 df['A/B_ment_ratio'] =(df['A_ment_ratio'] > df['B_ment_ratio'])
 df['A/B_rt_ratio'] =(df['A_rt_ratio'] > df['B_rt_ratio'])
 df['nf1_diff'] =(df['A_network_feature_1'] > df['B_network_feature_1'])
 df['nf2_diff'] =(df['A_network_feature_2'] > df['B_network_feature_2'])
 df['nf3_diff'] =(df['A_network_feature_3'] > df['B_network_feature_3'])
 df['nf3_ratio'] = df['A_network_feature_3'] / df['B_network_feature_3']
df['nf2_ratio'] = df['A_network_feature_2'] / df['B_network_feature_2']
df['nf1_ratio'] = df['A_network_feature_1'] / df['B_network_feature_1']
return(pre_pro(df)) 
<categorify>
test_df['Seed_diff'] = test_df['Seed1'] - test_df['Seed2']
test_df['ScoreT_diff'] = test_df['ScoreT1'] - test_df['ScoreT2']
test_df = test_df.drop(['ID', 'Pred', 'Season', 'WTeamID', 'LTeamID'], axis=1)
 test_df
fe_train = feat_eng(train.copy())
 fe_test = feat_eng(test.copy() )<prepare_x_and_y>
X = tourney_result_Final1.drop('result', axis=1)
 y = tourney_result_Final1.result
train_df = fe_train
test_df = fe_test
y_train = np.array(train_df['Choice'] )<define_variables>
df = pd.concat([X, test_df], axis=0, sort=False ).reset_index(drop=True)
 df_log = pd.DataFrame(
 preprocessing.MinMaxScaler().fit_transform(df),
columns=df.columns,
index=df.index 
)
 train_log, test_log = df_log.iloc[:len(X),:], df_log.iloc[len(X):,:].reset_index(drop=True )
target = 'Choice'
predictors = train_df.columns.values.tolist() [1:]<init_hyperparams>
logreg = LogisticRegression()
logreg.fit(train_log, y)
 coeff_logreg = pd.DataFrame(train_log.columns.delete(0))
coeff_logreg.columns = ['feature']
coeff_logreg["score_logreg"] = pd.Series(logreg.coef_[0])
 coeff_logreg.sort_values(by='score_logreg', ascending=False )
param_lgb = {
'feature_fraction': 0.4647875434283183,
'lambda_l1': 0.14487098904632512,
'lambda_l2': 0.9546002933329684,
'learning_rate': 0.050592093295320606,
'max_depth': int(round(7.696194993998026)) ,
'min_data_in_leaf': int(round(9.879507661608065)) ,
'min_gain_to_split': 0.7998292013880356,
'min_sum_hessian_in_leaf': 0.24962103361366683,
'num_leaves': int(round(2.854239951949671)) ,
'max_bin': 63,
'bagging_fraction': 1.0,
'bagging_freq': 5,
'save_binary': True,
'seed': 1965,
'feature_fraction_seed': 1965,
'bagging_seed': 1965,
'drop_seed': 1965,
'data_random_seed': 1965,
'objective': 'binary',
'boosting_type': 'gbdt',
'verbose': 1,
'metric': 'auc',
'is_unbalance': True,
'boost_from_average': False}<split>
y_logreg_train = logreg.predict(train_log)
 y_logreg_pred = logreg.predict_proba(test_log )
nfold = 20
skf = StratifiedKFold(n_splits=nfold, shuffle=True, random_state=2019)
 oof = np.zeros(len(fe_train))
predictions = np.zeros(( len(fe_test),nfold))
i = 1
for train_index, valid_index in skf.split(fe_train, fe_train.Choice.values):
print("
fold {}".format(i))
xg_train = lgb.Dataset(fe_train.iloc[train_index][predictors].values,
label=fe_train.iloc[train_index][target].values,
feature_name=predictors,
free_raw_data = False 
)
 xg_valid = lgb.Dataset(fe_train.iloc[valid_index][predictors].values,
label=fe_train.iloc[valid_index][target].values,
feature_name=predictors,
free_raw_data = False 
)
 clf = lgb.train(param_lgb, xg_train, 10000000, valid_sets = [xg_valid], verbose_eval=250, early_stopping_rounds = 100)
 oof[valid_index] = clf.predict(fe_train.iloc[valid_index][predictors].values, num_iteration=clf.best_iteration)
 predictions[:,i-1] += clf.predict(fe_test[predictors], num_iteration=clf.best_iteration)
 i = i + 1
print("
CV AUC: {:<0.8f}".format(metrics.roc_auc_score(fe_train.Choice.values, oof)) )<groupby>
clf = RandomForestClassifier(n_estimators=200,max_depth=50)
 clf.fit(train_log, y)
 clf_probs = clf.predict_proba(test_log )
lgb_bay = []
for i in range(len(predictions)) :
lgb_bay.append(predictions[i][-1] )<save_to_csv>
y_pred_df_random = pd.DataFrame(clf_probs)
 y_pred_1 = y_pred_df_random.iloc[:,[1]]
y_pred_df_random
submission = pd.read_csv('.. /input/sample_predictions.csv')
 submission['Choice'] = lgb_bay
submission.to_csv('sub.csv', index = False, header = True )<import_modules>
submission_df = pd.read_csv('.. /input/google-cloud-ncaa-march-madness-2020-division-1-womens-tournament/WSampleSubmissionStage1_2020.csv')
 submission_df['Pred'] = y_pred_1
submission_df
<set_options><EOS> 
submission_df.to_csv('submission_New8.csv', index=False )