Kaggle competition: Google Cloud & NCAA® ML Competition 2020-NCAAM
<SOS> metric: logloss Kaggle data source: google-cloud-ncaa-march-madness-2020-division-1-mens-tournament<import_modules>
py.init_notebook_mode(connected=True)
 pd.set_option('max_columns', 50 )
import matplotlib.pyplot as plt<data_type_conversions>
datadir = Path('/kaggle/input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament')
 stage1dir = datadir/'MDataFiles_Stage1'
datetime_str = '01/22/20 00:00:00'
datetime_object = datetime.strptime(datetime_str, '%m/%d/%y %H:%M:%S')
 data['days']=pd.to_datetime(data['Date'] ).sub(datetime_object)/np.timedelta64(1, 'D')
 test['days']=pd.to_datetime(test['Date'] ).sub(datetime_object)/np.timedelta64(1, 'D' )<sort_values>
teams_df = pd.read_csv(stage1dir/'MTeams.csv')
 print('teams_df', teams_df.shape)
 teams_df.head()
data.loc[(data['Province_State']=='')&(data['Country_Region']=='India'),:].sort_values(by="Date" )<import_modules>
seasons_df = pd.read_csv(stage1dir/'MSeasons.csv' )
from statsmodels.tsa.arima_model import ARIMA 
<data_type_conversions>
tourney_seeds_df = pd.read_csv(stage1dir/'MNCAATourneySeeds.csv')
 tourney_seeds_df
data['Date']=pd.to_datetime(data['Date'])
 test['Date']=pd.to_datetime(test['Date'] )<create_dataframe>
regular_season_results_df = pd.read_csv(stage1dir/'MRegularSeasonCompactResults.csv')
 tournament_results_df = pd.read_csv(stage1dir/'MNCAATourneyCompactResults.csv' )
pd.DataFrame(data.loc[data['Country_Region']=='Afghanistan',['ConfirmedCases']] ).reset_index(drop=True )<count_missing_values>
sample_submission = pd.read_csv(datadir/'MSampleSubmissionStage1_2020.csv')
 sample_submission
data.isna().sum(axis=0 )<filter>
regular_season_detailed_results_df = pd.read_csv(stage1dir/'MRegularSeasonDetailedResults.csv')
 tournament_detailed_results_df = pd.read_csv(stage1dir/'MNCAATourneyDetailedResults.csv' )
data['ConfirmedCases'][data['Country_Region']==''][51:]<count_values>
cities_df = pd.read_csv(stage1dir/'Cities.csv')
 mgame_cities_df = pd.read_csv(stage1dir/'MGameCities.csv' )
data['ConfirmedCases'][data['Country_Region']=='India'].value_counts()<create_dataframe>
massey_df = pd.read_csv(stage1dir/'MMasseyOrdinals.csv')
 massey_df
pd.DataFrame(data.loc[data['Country_Region']=='India',['ConfirmedCases']] )<data_type_conversions>
event2015_df = pd.read_csv(datadir/'MEvents2015.csv')
datetime_str = '03/22/20 00:00:00'
datetime_object = datetime.strptime(datetime_str, '%m/%d/%y %H:%M:%S' )<import_modules>
players_df = pd.read_csv(datadir/'MPlayers.csv')
 players_df
from datetime import timedelta<import_modules>
team_coaches_df = pd.read_csv(stage1dir/'MTeamCoaches.csv')
 print('team_coaches_df', team_coaches_df.shape)
 team_coaches_df.iloc[80:85]
import math<compute_test_metric>
conferences_df = pd.read_csv(stage1dir/'Conferences.csv')
 team_conferences_df = pd.read_csv(stage1dir/'MTeamConferences.csv' )
def rmsle(y, y_pred):
assert len(y)== len(y_pred)
 terms_to_sum = [(math.log(y_pred[i] + 1)- math.log(y[i] + 1)) ** 2.0 for i,pred in enumerate(y_pred)]
return(sum(terms_to_sum)*(1.0/len(y)))** 0.5<train_model>
team_conferences_df[team_conferences_df['TeamID'] == 1102]
def evaluate_arima_model(X,forecast_days, arima_order):
X=[x for x in X]
train_size = int(len(X)* 0.9)
 train, test1 = X[0:train_size], X[train_size:]
history=train
model = ARIMA(history, order=arima_order)
 model_fit = model.fit(disp=0)
 predictions = list()
predictions =model_fit.forecast(steps=len(test1)) [0]
model = ARIMA(X, order=arima_order)
 model_fit = model.fit(disp=0)
 if np.isnan(model_fit.forecast(steps=forecast_days)[0] ).sum() >0:
return float('inf')
 error = rmsle(test1, predictions)
 return error<find_best_params>
conference_tourney_games_df = pd.read_csv(stage1dir/'MConferenceTourneyGames.csv')
 conference_tourney_games_df
def evaluate_models(dataset,forcast_days, p_values, d_values, q_values):
best_score, best_cfg = float("inf"),(0,0,0)
 for p in p_values:
for d in d_values:
for q in q_values:
order =(p,d,q)
 try:
mse = evaluate_arima_model(dataset,forcast_days, order)
 if mse < best_score:
best_score, best_cfg = mse, order
except:
continue
print('Best ARIMA%s MSE=%.3f' %(best_cfg, best_score))
return best_cfg, best_score<set_options>
secondary_tourney_teams_df = pd.read_csv(stage1dir/'MSecondaryTourneyTeams.csv')
 secondary_tourney_teams_df
warnings.filterwarnings('ignore' )<feature_engineering>
secondary_tourney_results_df = pd.read_csv(stage1dir/'MSecondaryTourneyCompactResults.csv')
 secondary_tourney_results_df
test['ConfirmedCases']=0
test['Fatalities']=0<filter>
team_spellings_df = pd.read_csv(stage1dir/'MTeamSpellings.csv',encoding="cp1252" )
sliced_data=data.loc[(data['Province_State']=='')&(data['Country_Region']=='India'),:]<import_modules>
tourney_slots_df = pd.read_csv(stage1dir/'MNCAATourneySlots.csv')
 tourney_seed_round_slots_df = pd.read_csv(stage1dir/'MNCAATourneySeedRoundSlots.csv' )
from pandas import read_csv
from pandas import datetime
from matplotlib import pyplot
from statsmodels.tsa.arima_model import ARIMA
from sklearn.metrics import mean_squared_error
from math import sqrt
from time import time
from sklearn.metrics import mean_squared_error<remove_duplicates>
tourney_slots_df[(tourney_slots_df['Season'] == 1985)&(tourney_slots_df['Slot'].str.startswith('R1W')) ]
country='India'
state=''
sliced_data=data.loc[(data['Province_State']==state)&(data['Country_Region']==country),:]
test_sliced=test.loc[(test['Province_State']==state)&(test['Country_Region']==country),:]
sliced_data=sliced_data.drop_duplicates()
sliced_data=sliced_data.reset_index(drop=True)
 sliced_data=sliced_data.sort_values(by='Date')
 if sliced_data.loc[sliced_data['ConfirmedCases']>0,:].shape[0]>0:
sliced_data=sliced_data.loc[sliced_data['ConfirmedCases']>0,:]
sliced_data=sliced_data.reset_index(drop=True)
 max_date_train=sliced_data['Date'].max()
max_date_test=test_sliced['Date'].max()
forcast_days=int(( max_date_test-max_date_train)/np.timedelta64(1, 'D'))
history=sliced_data['ConfirmedCases'].to_list()
if len(history)==1:
history.append(history[0])
 best_cfg,best_score=evaluate_models(history,forcast_days,range(10),range(7),range(7))
preds=[]
model = ARIMA(history, order=best_cfg)
 model_fit = model.fit(disp=0)
 preds=model_fit.forecast(steps=forcast_days)[0]
preds=[round(p)if p>0 else 0 for p in preds]
dates=[max_date_train+timedelta(days=day+1)for day in range(forcast_days)]
predictions=pd.DataFrame()
predictions['Date']=dates
predictions['ConfirmedCases']=preds
test_sliced=test_sliced.merge(sliced_data[['Date','ConfirmedCases']], on='Date',how='left')
 test_sliced['ConfirmedCases']=test_sliced['ConfirmedCases_y']
del test_sliced['ConfirmedCases_y']
del test_sliced['ConfirmedCases_x']
test_sliced=test_sliced.merge(predictions, on='Date',how='left')
 test_sliced['ConfirmedCases_x'][test_sliced['ConfirmedCases_x'].isna() ]=test_sliced['ConfirmedCases_y'][test_sliced['ConfirmedCases_x'].isna() ]
test_sliced['ConfirmedCases']=test_sliced['ConfirmedCases_x']
del test_sliced['ConfirmedCases_y']
del test_sliced['ConfirmedCases_x']
sliced_data_bck=sliced_data.copy()
if sliced_data.loc[sliced_data['Fatalities']>0,:].shape[0]>0:
sliced_data=sliced_data.loc[sliced_data['Fatalities']>0,:]
sliced_data=sliced_data.reset_index(drop=True)
 max_date_train=sliced_data['Date'].max()
max_date_test=test_sliced['Date'].max()
forcast_days=int(( max_date_test-max_date_train)/np.timedelta64(1, 'D'))
history=sliced_data['Fatalities'].to_list()
if len(history)==1:
history.append(history[0])
 best_cfg,best_score=evaluate_models(history,forcast_days,range(5),range(5),range(5))
preds=[]
model=None
model = ARIMA(history, order=best_cfg)
 model_fit = model.fit(disp=0)
 preds=model_fit.forecast(steps=forcast_days)[0]
preds=[round(p)if p>0 else 0 for p in preds]
dates=[max_date_train+timedelta(days=day+1)for day in range(forcast_days)]
predictions_f=pd.DataFrame()
predictions_f['Date']=dates
predictions_f['Fatalities']=preds
test_sliced=test_sliced.merge(sliced_data_bck[['Date','Fatalities']], on='Date',how='left')
 test_sliced['Fatalities']=test_sliced['Fatalities_y']
del test_sliced['Fatalities_y']
del test_sliced['Fatalities_x']
test_sliced=test_sliced.merge(predictions_f, on='Date',how='left')
 test_sliced['Fatalities_x'][test_sliced['Fatalities_x'].isna() ]=test_sliced['Fatalities_y'][test_sliced['Fatalities_x'].isna() ]
test_sliced['Fatalities']=test_sliced['Fatalities_x']
del test_sliced['Fatalities_y']
del test_sliced['Fatalities_x']
test=test.merge(test_sliced,on='ForecastId',how='left')
 test['ConfirmedCases_x'][test['ConfirmedCases_y'].notna() ]=test['ConfirmedCases_y'][test['ConfirmedCases_y'].notna() ]
test['Fatalities_x'][test['Fatalities_y'].notna() ]=test['Fatalities_y'][test['Fatalities_y'].notna() ]
new_cols=[]
for col in test.columns:
if col[-2:]=='_y':
del test[col]
elif col[-2:]=='_x':
new_cols.append(col[:-2])
 else:
new_cols.append(col)
 test.columns=new_cols
test.loc[(test['Province_State']==state)&(test['Country_Region']==country),:].head()
plt.plot('Date', 'ConfirmedCases', data=sliced_data, color='blue', linewidth=2)
 plt.plot('Date','ConfirmedCases',data=test_sliced,color='orange',linewidth=2)
 plt.plot('Date', 'Fatalities', data=sliced_data, color='purple', linewidth=2)
 plt.plot('Date','Fatalities',data=test_sliced,color='red',linewidth=2)
 plt.show()<filter>
tournament_results2015_df = tournament_results_df.query("Season >= 2015")
 tournament_results2015_df
test.loc[(test['Province_State']==state)&(test['Country_Region']==country),['Country_Region','Date','ConfirmedCases','Fatalities']]<feature_engineering>
for key, row in tournament_results2015_df.iterrows() :
if row['WTeamID'] < row['LTeamID']:
id_name = str(row['Season'])+ '_' + str(row['WTeamID'])+ '_' + str(row['LTeamID'])
 sample_submission.loc[sample_submission['ID'] == id_name, 'Pred'] = 1.0
else:
id_name = str(row['Season'])+ '_' + str(row['LTeamID'])+ '_' + str(row['WTeamID'])
 sample_submission.loc[sample_submission['ID'] == id_name, 'Pred'] = 0.0
<remove_duplicates><EOS> 
sample_submission.to_csv('submission.csv', index=False )