Kaggle competition: Google Cloud & NCAA® ML Competition 2018-Women's
loc_group = ["Province_State", "Country_Region"]
def preprocess(df,datecol):
df["Date"] = df[datecol].astype("datetime64[ms]")
 for col in loc_group:
df[col].fillna("none", inplace=True)
 return df
df= preprocess(df,"Date" )<define_variables>
df_seeds = pd.read_csv('.. /input/WNCAATourneySeeds_SampleTourney2018.csv')
 df_tour = pd.read_csv('.. /input/WRegularSeasonCompactResults_PrelimData2018.csv' )
TRAIN_X = datetime.strptime('20-03-11', '%y-%m-%d' )<define_variables>
df_seeds['seed_int'] = df_seeds['Seed'].apply(lambda x : int(x[1:3]))
df_winseeds = df_seeds.loc[:, ['TeamID', 'Season', 'seed_int']].rename(columns={'TeamID':'WTeamID', 'seed_int':'WSeed'})
 df_lossseeds = df_seeds.loc[:, ['TeamID', 'Season', 'seed_int']].rename(columns={'TeamID':'LTeamID', 'seed_int':'LSeed'})
 df_dummy = pd.merge(left=df_tour, right=df_winseeds, how='left', on=['Season', 'WTeamID'])
 df_concat = pd.merge(left=df_dummy, right=df_lossseeds, on=['Season', 'LTeamID'] )
TEST_END = df["Date"].max()
TRAIN_LAST = datetime.strptime('20-03-23', '%y-%m-%d')
 TRAIN_FIRST = df["Date"].max() - timedelta(days=5 )<sort_values>
df_concat['DiffSeed'] = df_concat[['LSeed', 'WSeed']].apply(lambda x : 0 if x[0] == x[1] else 1, axis = 1 )
df=df.sort_values(['Province_State','Country_Region','Date'] )<feature_engineering>
df_sample_sub = pd.read_csv('.. /input/WSampleSubmissionStage2.csv' )
TARGETS = ["ConfirmedCases", "Fatalities"]
for col in TARGETS:
df["prev_{}".format(col)] = df[col].shift()
df[f'delta_{col}'] = df[col]-df[f'prev_{col}']
df[f'rolling_{col}'] = df[f'delta_{col}'].rolling(window=3 ).mean()
df[f'div_rolling_{col}'] =(df[f'rolling_{col}']-df[f'rolling_{col}'].shift())/df[f'rolling_{col}'].shift()
df['div_rolling_ConfirmedCases']=df['div_rolling_ConfirmedCases'].replace([np.inf, -np.inf], np.nan ).fillna(0)
 df['div_rolling_Fatalities']=df['div_rolling_Fatalities'].replace([np.inf, -np.inf], np.nan ).fillna(0 )<filter>
df_sample_sub['Season'] = df_sample_sub['ID'].apply(lambda x : int(x.split('_')[0]))
df_sample_sub['TeamID1'] = df_sample_sub['ID'].apply(lambda x : int(x.split('_')[1]))
df_sample_sub['TeamID2'] = df_sample_sub['ID'].apply(lambda x : int(x.split('_')[2]))
df[df['Province_State']=="New York"]<feature_engineering>
winners = df_concat.rename(columns = { 'WTeamID' : 'TeamID1',
'LTeamID' : 'TeamID2',
'WScore' : 'Team1_Score',
'LScore' : 'Team2_Score'} ).drop(['WSeed', 'LSeed', 'WLoc'], axis = 1)
 winners['Result'] = 1.0
losers = df_concat.rename(columns = { 'WTeamID' : 'TeamID2',
'LTeamID' : 'TeamID1',
'WScore' : 'Team2_Score',
'LScore' : 'Team1_Score'} ).drop(['WSeed', 'LSeed', 'WLoc'], axis = 1)
 losers['Result'] = 0.0
train = pd.concat([winners, losers], axis = 0 ).reset_index(drop = True)
 train['Score_Ratio'] = train['Team1_Score'] / train['Team2_Score']
train['Score_Total'] = train['Team1_Score'] + train['Team2_Score']
train['Score_Pct'] = train['Team1_Score'] / train['Score_Total']
df = df[df.Date>df.Date.min() + timedelta(days=3)]<filter>
train_test_inner = pd.merge(train.loc[ train['Season'].isin([2018]), : ].reset_index(drop = True),
df_sample_sub.drop(['ID', 'Pred'], axis = 1),
on = ['Season', 'TeamID1', 'TeamID2'], how = 'inner' )
dfSmall = df[df.rolling_ConfirmedCases>100].reset_index()<groupby>
team1d_score_spread = train_test_inner.groupby(['Season', 'TeamID1'])[['Score_Ratio', 'Score_Pct']].median().reset_index() \ 
.set_index('Season' ).rename(columns = {'Score_Ratio' : 'Score_Ratio1', 'Score_Pct' : 'Score_Pct1'})
 team2d_score_spread = train_test_inner.groupby(['Season', 'TeamID2'])[['Score_Ratio', 'Score_Pct']].median().reset_index() \ 
.set_index('Season' ).rename(columns = {'Score_Ratio' : 'Score_Ratio2', 'Score_Pct' : 'Score_Pct2'})
 score_spread = team1d_score_spread.join(team2d_score_spread ).reset_index()
score_spread['Score_Ratio'] = score_spread[['Score_Ratio1', 'Score_Ratio2']].apply(lambda x :(x[0] *(x[1] ** -1.0)) , axis = 1)** 0.5
score_spread['Score_Pct'] = score_spread[['Score_Pct1', 'Score_Pct2']].apply(lambda x : 0.5*(x[0] ** -1.0)+ 0.5*(1.0 - x[1])** -1.0, axis = 1)** -1.0
score_spread.head()
MaxDate = dfSmall.loc[dfSmall.groupby(loc_group)['rolling_ConfirmedCases'].idxmax() ][['Province_State','Country_Region','Date']].reset_index()<merge>
X_train = train_test_inner.loc[:, ['Season', 'NumOT', 'Score_Ratio', 'Score_Pct']]
train_labels = train_test_inner['Result']
train_test_outer = pd.merge(train.loc[ train['Season'].isin([2014, 2015, 2016, 2017]), : ].reset_index(drop = True),
df_sample_sub.drop(['ID', 'Pred'], axis = 1),
on = ['Season', 'TeamID1', 'TeamID2'], how = 'outer')
 train_test_outer = train_test_outer.loc[ train_test_outer['Result'].isnull() ,
['TeamID1', 'TeamID2', 'Season']]
train_test_missing = pd.merge(pd.merge(score_spread.loc[:, ['TeamID1', 'TeamID2', 'Season', 'Score_Ratio', 'Score_Pct']],
train_test_outer, on = ['TeamID1', 'TeamID2', 'Season']),
num_ot.loc[:, ['TeamID1', 'TeamID2', 'Season', 'NumOT']],
on = ['TeamID1', 'TeamID2', 'Season'] )
df = df.merge(MaxDate, how = 'left',on=['Province_State','Country_Region'],suffixes=['','_Max'] )<groupby>
X_test = train_test_missing.loc[:, ['Season', 'NumOT', 'Score_Ratio', 'Score_Pct']]
n = X_train.shape[0]
train_test_merge = pd.concat([X_train, X_test], axis = 0 ).reset_index(drop = True)
 train_test_merge = pd.concat([pd.get_dummies(train_test_merge['Season'].astype(object)) ,
train_test_merge.drop('Season', axis = 1)], axis = 1)
 train_test_merge = pd.concat([pd.get_dummies(train_test_merge['NumOT'].astype(object)) ,
train_test_merge.drop('NumOT', axis = 1)], axis = 1)
 X_train = train_test_merge.loc[:(n - 1), :].reset_index(drop = True)
 X_test = train_test_merge.loc[n:, :].reset_index(drop = True )
MinDate = dfSmall.loc[dfSmall.groupby(loc_group)['rolling_ConfirmedCases'].idxmin() ][['Province_State','Country_Region','Date']].reset_index()<merge>
x_max = X_train.max()
x_min = X_train.min()
X_train =(X_train - x_min)/(x_max - x_min + 1e-14)
 X_test =(X_test - x_min)/(x_max - x_min + 1e-14 )
df = df.merge(MinDate, how = 'left',on=['Province_State','Country_Region'],suffixes=['','_Min'] )<feature_engineering>
model = LogisticRegressionCV(cv=25)
 model.fit(X_train, train_labels )
df['Date_Max'] = df['Date_Max'].fillna(TEST_END)
<feature_engineering>
train_test_inner['Pred1'] = model.predict_proba(X_train)[:,1]
train_test_missing['Pred1'] = model.predict_proba(X_test)[:,1]
df['Date_Min'] =df["Date_Max"] - timedelta(days=5 )<remove_duplicates>
sub = pd.merge(df_sample_sub,
pd.concat([train_test_missing.loc[:, ['Season', 'TeamID1', 'TeamID2', 'Pred1']],
train_test_inner.loc[:, ['Season', 'TeamID1', 'TeamID2', 'Pred1']] ],
axis = 0 ).reset_index(drop = True),
on = ['Season', 'TeamID1', 'TeamID2'], how = 'outer' )
df[['Date_Max','Date_Min']].drop_duplicates()<data_type_conversions>
team1_probs = sub.groupby('TeamID1')['Pred1'].apply(lambda x :(x ** -1.0 ).mean() ** -1.0 ).fillna(0.5 ).to_dict()
team2_probs = sub.groupby('TeamID2')['Pred1'].apply(lambda x :(x ** -1.0 ).mean() ** -1.0 ).fillna(0.5 ).to_dict()
df['Days'] =(df['Date']-df['Date_Min'].min() ).dt.days.astype('int16' )<feature_engineering>
sub['Pred'] = sub[['TeamID1', 'TeamID2','Pred1']]\ 
.apply(lambda x : team1_probs.get(x[0])*(1 - team2_probs.get(x[1])) if np.isnan(x[2])else x[2],
axis = 1 )
<feature_engineering><EOS> 
sub = sub.drop_duplicates(subset=["ID"], keep='first')
 sub[['ID', 'Pred']].to_csv('sub.csv', index = False )