Kaggle competition: COVID19 Global Forecasting (Week 3)
<SOS> metric: MCRMSLE Kaggle data source: covid19-global-forecasting-week-3<load_from_csv>
warnings.filterwarnings('ignore')
 daybasecount = 4
baseday = 98 - float(daybasecount-1)/2.
 exponent = 1./float(daybasecount)
 fatalityBaseDayShift = 10
maxincrease = 140
maxDeadPrDay = 1500
path = '.. /input/google-cloud-ncaa-march-madness-2020-division-1-womens-tournament/'
tourney_result = pd.read_csv(path + 'WDataFiles_Stage1/WNCAATourneyCompactResults.csv')
 tourney_seed = pd.read_csv(path + 'WDataFiles_Stage1/WNCAATourneySeeds.csv')
 season_result = pd.read_csv(path + 'WDataFiles_Stage1/WRegularSeasonCompactResults.csv' )<load_from_csv>
dftrain = pd.read_csv('.. /input/covid19-global-forecasting-week-3/train.csv', parse_dates=['Date'] ).sort_values(by=['Country_Region', 'Date'])
 dftest = pd.read_csv('.. /input/covid19-global-forecasting-week-3/test.csv', parse_dates=['Date'] ).sort_values(by=['Country_Region', 'Date'])
 ppp_tabel = pd.read_csv('.. /input/country-ppp/Country_PPP.csv', sep='\s+')
 ppp_tabel.drop('Id', 1,inplace=True)
 ppp_tabel = ppp_tabel.append({'Country' : 'Burma' , 'ppp' : 8000} , ignore_index=True)
 ppp_tabel = ppp_tabel.append({'Country' : 'MS_Zaandam' , 'ppp' : 40000} , ignore_index=True)
 ppp_tabel = ppp_tabel.append({'Country' : 'West_Bank_and_Gaza' , 'ppp' : 20000} , ignore_index=True)
 ppp_tabel["Country"].replace('_',' ', regex=True,inplace=True)
 ppp_tabel["Country"].replace('United States','US', regex=True,inplace=True)
 ppp_tabel.rename(columns={'Country':'Country_Region'},inplace=True)
 ppp_tabel.sort_values('Country_Region',inplace=True )
test_df= pd.read_csv(path +'WSampleSubmissionStage1_2020.csv' )<drop_column>
dftrain['Dayofyear'] = dftrain['Date'].dt.dayofyear
dftest['Dayofyear'] = dftest['Date'].dt.dayofyear
dftest['Expo'] = dftest['Dayofyear']-baseday
print(dftrain.tail(5))
dftest = dftest.merge(dftrain[['Country_Region','Province_State','Date','ConfirmedCases','Fatalities']]
, on=['Country_Region','Province_State','Date'], how='left', indicator=True)
tourney_result = tourney_result.drop(['DayNum', 'WScore', 'LScore', 'WLoc', 'NumOT'], axis=1)
 tourney_result<merge>
grouped=dftrain.groupby(['Country_Region','Province_State'] ).tail(daybasecount*2)
 grouped=grouped.groupby(['Country_Region','Province_State'] ).head(daybasecount)
 grouped.drop(['FatalityBasis'],axis=1,inplace=True)
 to_sum = ['NewCases','NewFatalities']
grouped1 = grouped.groupby(['Country_Region'])[to_sum].sum()
grouped1.rename(columns={'NewCases':'NewCases1','NewFatalities':'NewFatalities1'}, inplace=True)
 grouped = pd.merge(grouped1, grouped2, on=['Country_Region'])
 grouped['CasesIncreasePct'] = 100*(grouped['NewCases2']/grouped['NewCases1']-1)
 mask = grouped['CasesIncreasePct'] > maxincrease
grouped.loc[mask,'CasesIncreasePct'] = maxincrease
mask = grouped['CasesIncreasePct'] < 0
grouped.loc[mask,'CasesIncreasePct'] = 0
mask = grouped['CasesIncreasePct'].isnull()
grouped.loc[mask,'CasesIncreasePct'] = 0
grouped['Factor'] =(grouped['CasesIncreasePct']/100+1)**exponent
grouped = pd.merge(grouped, ppp_tabel, on=['Country_Region'])
 grouped['ppp'] = grouped['ppp']/10000.
 if False:
mask =(grouped['FatalityPct2'] > 9)&(grouped['ppp'] <= 1)
 grouped.loc[mask,'FatalityPct2'] = 5
mask =(grouped['FatalityPct2'] < 5)&(grouped['ppp'] <= 1)
 grouped.loc[mask,'FatalityPct2'] = 5
mask =(grouped['FatalityPct2'] > 6)&(grouped['ppp'] >= 7)
 grouped.loc[mask,'FatalityPct2'] = 6
mask =(grouped['FatalityPct2'] < 1.5)&(grouped['ppp'] >= 7)
 grouped.loc[mask,'FatalityPct2'] = 1.5
mask =(grouped['FatalityPct2'] >(9.5 - 0.43*grouped['ppp'])) &(grouped['ppp'] > 1)&(grouped['ppp'] < 7)
 grouped.loc[mask,'FatalityPct2'] =(9.5 - 0.43*grouped['ppp'])
 mask =(grouped['FatalityPct2'] <(5.6 - 0.5*grouped['ppp'])) &(grouped['ppp'] > 1)&(grouped['ppp'] < 7)
 grouped.loc[mask,'FatalityPct2'] =(5.6 - 0.5*grouped['ppp'])
 mask =(grouped['FatalityPct2'].isnull())&(grouped['ppp'] <= 1)
 grouped.loc[mask,'FatalityPct2'] = 7
mask =(grouped['FatalityPct2'].isnull())&(grouped['ppp'] >= 7)
 grouped.loc[mask,'FatalityPct2'] = 4
mask =(grouped['FatalityPct2'].isnull())&(grouped['ppp'] > 1)&(grouped['ppp'] < 7)
 grouped.loc[mask,'FatalityPct2'] =(7.5 - 0.5*grouped['ppp'])
 else:
mask =(grouped['FatalityPct2'] > 4)&(grouped['ppp'] <= 1)
 grouped.loc[mask,'FatalityPct2'] = 3
mask =(grouped['FatalityPct2'] < 1)&(grouped['ppp'] <= 1)
 grouped.loc[mask,'FatalityPct2'] = 2
mask =(grouped['FatalityPct2'] > 1.5)&(grouped['ppp'] >= 7)
 grouped.loc[mask,'FatalityPct2'] = 1.5
mask =(grouped['FatalityPct2'] < 0.5)&(grouped['ppp'] >= 7)
 grouped.loc[mask,'FatalityPct2'] = 0.5
mask =(grouped['FatalityPct2'] >(4.5 - 0.43*grouped['ppp'])) &(grouped['ppp'] > 1)&(grouped['ppp'] < 7)
 grouped.loc[mask,'FatalityPct2'] =(4.5 - 0.43*grouped['ppp'])
 mask =(grouped['FatalityPct2'] <(1.1 - 0.1*grouped['ppp'])) &(grouped['ppp'] > 1)&(grouped['ppp'] < 7)
 grouped.loc[mask,'FatalityPct2'] =(1.1 - 0.1*grouped['ppp'])
 mask =(grouped['FatalityPct2'].isnull())&(grouped['ppp'] <= 1)
 grouped.loc[mask,'FatalityPct2'] = 3
mask =(grouped['FatalityPct2'].isnull())&(grouped['ppp'] >= 7)
 grouped.loc[mask,'FatalityPct2'] = 1
mask =(grouped['FatalityPct2'].isnull())&(grouped['ppp'] > 1)&(grouped['ppp'] < 7)
 grouped.loc[mask,'FatalityPct2'] =(2.6 - 0.23*grouped['ppp'] )
tourney_result = pd.merge(tourney_result, tourney_seed, left_on=['Season', 'WTeamID'], right_on=['Season', 'TeamID'], how='left')
 tourney_result.rename(columns={'Seed':'WSeed'}, inplace=True)
 tourney_result = tourney_result.drop('TeamID', axis=1)
 tourney_result = pd.merge(tourney_result, tourney_seed, left_on=['Season', 'LTeamID'], right_on=['Season', 'TeamID'], how='left')
 tourney_result.rename(columns={'Seed':'LSeed'}, inplace=True)
 tourney_result = tourney_result.drop('TeamID', axis=1)
 tourney_result<categorify>
dftest.drop('_merge',axis=1,inplace= True)
 dftest = dftest.merge(grouped[['Country_Region','FatalityPct2','Factor']], on=['Country_Region'], how='left')
 dftest = dftest.merge(grouped_gem[['Province_State','Country_Region','ConfirmedCases_base','ConfirmedCases_init','NewCases_base','Fatalities_init','FatalityBasis']], on=['Province_State','Country_Region'], how='left')
def get_seed(x):
return int(x[1:3])
 tourney_result['WSeed'] = tourney_result['WSeed'].map(lambda x: get_seed(x))
tourney_result['LSeed'] = tourney_result['LSeed'].map(lambda x: get_seed(x))
tourney_result<concatenate>
dftest['ConfirmedCases_shift'] = dftest.groupby(['Country_Region', 'Province_State'])[['ConfirmedCases']].transform(lambda x: x.shift(1))
mask = dftest['ConfirmedCases'].isnull()
dftest.loc[mask,'NewCases'] = dftest.loc[mask,'NewCases_base']*(dftest.loc[mask,'Factor']**dftest.loc[mask,'Expo'])
 dftest['NewCases_cum'] = dftest.groupby(['Country_Region', 'Province_State'])[['NewCases']].cumsum()
dftest.loc[mask,'ConfirmedCases'] = dftest.loc[mask,'ConfirmedCases_init'] + dftest.loc[mask,'NewCases_cum']
mask3 = dftest['ConfirmedCases'] > 400000
dftest.loc[mask3,'FatalityPct2'] = dftest.loc[mask3,'FatalityPct2']*0.7
mask4 = dftest['ConfirmedCases'] > 800000
dftest.loc[mask4,'FatalityPct2'] = dftest.loc[mask4,'FatalityPct2']*0.7
dftest['FatalityBasis'] = dftest.groupby(['Country_Region', 'Province_State'])[
['ConfirmedCases']].transform(lambda x: x.shift(10))
dftest.loc[mask,'NewFatalities'] = dftest.loc[mask,'FatalityBasis'] * dftest.loc[mask,'FatalityPct2']/100
mask2 = dftest['NewFatalities'] > maxDeadPrDay
dftest.loc[mask2,'NewFatalities'] = maxDeadPrDay
dftest['NewFatalities_cum'] = dftest.groupby(['Country_Region', 'Province_State'])[['NewFatalities']].cumsum()
dftest.loc[mask,'Fatalities'] = dftest.loc[mask,'Fatalities_init'] + dftest.loc[mask,'NewFatalities_cum']
season_win_result = season_result[['Season', 'WTeamID', 'WScore']]
season_lose_result = season_result[['Season', 'LTeamID', 'LScore']]
season_win_result.rename(columns={'WTeamID':'TeamID', 'WScore':'Score'}, inplace=True)
 season_lose_result.rename(columns={'LTeamID':'TeamID', 'LScore':'Score'}, inplace=True)
 season_result = pd.concat(( season_win_result, season_lose_result)).reset_index(drop=True)
 season_result<groupby>
dftest.drop(['Dayofyear',
'Expo','FatalityPct2', 'Factor',
'ConfirmedCases_base', 'ConfirmedCases_init',
'NewCases_base', 'Fatalities_init', 'FatalityBasis',
'ConfirmedCases_shift',
'NewCases', 'NewCases_cum', 'NewFatalities','NewFatalities_cum'],axis=1,inplace=True)
 final = dftest.groupby(['Country_Region','Province_State'] ).tail(1)
 dftest.drop(['Province_State'],axis=1,inplace=True)
 dftest.rename(columns={'Province_State_orig':'Province_State'},inplace=True )
season_score = season_result.groupby(['Season', 'TeamID'])['Score'].sum().reset_index()
season_score<merge>
mask = dftest["ConfirmedCases"].isnull()
print(mask.sum())
 errors = dftest.loc[mask]
print(errors)
 mask = dftest["Fatalities"].isnull()
print(mask.sum())
 errors = dftest.loc[mask]
print(errors)
 dftest.drop(['Province_State','Country_Region','Date'],axis=1,inplace=True)
 print("dftest columns =",dftest.columns)
<drop_column><EOS> 
dftest.ForecastId = dftest.ForecastId.astype('int')
 dftest['ConfirmedCases'] = dftest['ConfirmedCases'].round().astype(int)
 dftest['Fatalities'] = dftest['Fatalities'].round().astype(int)
 dftest.to_csv('submission.csv', index=False)