Kaggle competition: COVID19 Global Forecasting (Week 2)
<SOS> metric: MCRMSLE Kaggle data source: covid19-global-forecasting-(week-2)<filter>
warnings.filterwarnings('ignore' )
head = df_train.loc[(data["parentesco1"] ==1)&(data["Target"].notnull())].copy()<count_values>
dftrain = pd.read_csv('.. /input/covid19-global-forecasting-week-2/train.csv', parse_dates=['Date'] ).sort_values(by=['Country_Region', 'Date'] ).fillna('None')
 dftest = pd.read_csv('.. /input/covid19-global-forecasting-week-2/test.csv', parse_dates=['Date'] ).sort_values(by=['Country_Region', 'Date'] ).fillna('None')
 dfsubm = pd.read_csv('.. /input/covid19-global-forecasting-week-2/submission.csv' )
distrib_target = head['Target'].value_counts().reset_index().rename(columns = {'index' : 'level' } )<categorify>
confirmed = pd.read_csv('https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_covid19_confirmed_global.csv' ).sort_values(by='Country/Region')
 deaths = pd.read_csv('https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_covid19_deaths_global.csv')
 recovered = pd.read_csv('https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_covid19_recovered_global.csv' )
distrib_target['Household_type'] = distrib_target['level'].map(poverty_mapping )<define_variables>
confirmed['Country_Region'] = confirmed['Country/Region']
confirmed['Province_State'] = confirmed['Province/State']
confirmed.head()
ids = ['Id', 'idhogar']
target = ['Target']<define_variables>
dftrain = dftrain.join(confirmed[['Country_Region', 'Province_State', 'Lat', 'Long']].set_index(['Province_State', 'Country_Region']), on=['Province_State', 'Country_Region'] )
ind_bool = ['v18q', 'dis', 'male', 'female', 'estadocivil1', 'estadocivil2', 'estadocivil3',
'estadocivil4', 'estadocivil5', 'estadocivil6', 'estadocivil7',
'parentesco1', 'parentesco2', 'parentesco3', 'parentesco4', 'parentesco5',
'parentesco6', 'parentesco7', 'parentesco8', 'parentesco9', 'parentesco10',
'parentesco11', 'parentesco12', 'instlevel1', 'instlevel2', 'instlevel3',
'instlevel4', 'instlevel5', 'instlevel6', 'instlevel7', 'instlevel8',
'instlevel9', 'mobilephone']
ind_ordered = ['rez_esc', 'escolari', 'age']<define_variables>
dftrain['Dayofyear'] = dftrain['Date'].dt.dayofyear
dftest['Dayofyear'] = dftest['Date'].dt.dayofyear
hh_bool = ['hacdor', 'hacapo', 'v14a', 'refrig', 'paredblolad', 'paredzocalo',
'paredpreb','pisocemento', 'pareddes', 'paredmad',
'paredzinc', 'paredfibras', 'paredother', 'pisomoscer', 'pisoother',
'pisonatur', 'pisonotiene', 'pisomadera',
'techozinc', 'techoentrepiso', 'techocane', 'techootro', 'cielorazo',
'abastaguadentro', 'abastaguafuera', 'abastaguano',
'public', 'planpri', 'noelec', 'coopele', 'sanitario1',
'sanitario2', 'sanitario3', 'sanitario5', 'sanitario6',
'energcocinar1', 'energcocinar2', 'energcocinar3', 'energcocinar4',
'elimbasu1', 'elimbasu2', 'elimbasu3', 'elimbasu4',
'elimbasu5', 'elimbasu6', 'epared1', 'epared2', 'epared3',
'etecho1', 'etecho2', 'etecho3', 'eviv1', 'eviv2', 'eviv3',
'tipovivi1', 'tipovivi2', 'tipovivi3', 'tipovivi4', 'tipovivi5',
'computer', 'television', 'lugar1', 'lugar2', 'lugar3',
'lugar4', 'lugar5', 'lugar6', 'area1', 'area2']
hh_ordered = [ 'rooms', 'r4h1', 'r4h2', 'r4h3', 'r4m1','r4m2','r4m3', 'r4t1', 'r4t2',
'r4t3', 'v18q1', 'tamhog','tamviv','hhsize','hogar_nin',
'hogar_adul','hogar_mayor','hogar_total', 'bedrooms', 'qmobilephone']
hh_cont = ['v2a1', 'dependency', 'edjefe', 'edjefa', 'meaneduc', 'overcrowding'] 
<define_variables>
def transpose_df(df):
df = df.drop(['Lat','Long'],axis=1 ).groupby('Country/Region' ).sum().T
df.index = pd.to_datetime(df.index)
 return df
sqr_ = ['SQBescolari', 'SQBage', 'SQBhogar_total', 'SQBedjefe',
'SQBhogar_nin', 'SQBovercrowding', 'SQBdependency', 'SQBmeaned', 'agesq']<define_variables>
confirmedT = transpose_df(confirmed)
 deathsT = transpose_df(deaths)
 recoveredT = transpose_df(recovered)
 mortalityT = deathsT/confirmedT
head = data.loc[(data["parentesco1"] ==1), ids + hh_bool + hh_ordered + hh_cont + target].copy()<drop_column>
def add_day(df):
df['Date'] = df.index
df['Dayofyear'] = df['Date'].dt.dayofyear
return df
head.drop(labels = 'area2', axis = 1, inplace = True )<feature_engineering>
confirmedT, deathsT, recoveredT, mortalityT = add_day(confirmedT), add_day(deathsT), add_day(recoveredT), add_day(mortalityT )
head['epared_argmax'] = np.argmax(np.array(head[['epared1', 'epared2', 'epared3']]),
axis = 1 )<feature_engineering>
allcountries_ordered = confirmed.set_index(['Country_Region'] ).iloc[:,-2].sort_values(ascending=False ).index.tolist()
head['etecho_argmax'] = np.argmax(np.array(head[['etecho1', 'etecho2', 'etecho3']]),
axis = 1 )<prepare_output>
confirmed.set_index(['Country_Region'] ).iloc[:,-2].sort_values(ascending=False ).to_csv('confirmed_countries.csv' )
head['eviv_argmax'] = np.argmax(np.array(head[['eviv1', 'eviv2', 'eviv3']]),
axis = 1 )<prepare_output>
def df_day1(df, confirmed):
df_day1 = pd.DataFrame({'Days since 100 cases' : np.arange(1000)} ).set_index('Days since 100 cases')
 countries_df = df.columns.tolist() [:-2]
countries_conf = confirmed.columns.tolist() [:-2]
for ic, country in enumerate(countries_df):
for ic2, country2 in enumerate(countries_conf):
if country == country2:
dfsub = df[confirmed[country] > 100.][country]
df_day1[country] = np.nan
df_day1.loc[:len(dfsub)-1,country] =(dfsub ).tolist()
df_day1 = df_day1.dropna(how='all')
 return df_day1
head['abastagua_argmax'] = np.argmax(np.array(head[[ 'abastaguano', 'abastaguafuera','abastaguadentro']]),
axis = 1 )<prepare_output>
confirmed_day1 = df_day1(confirmedT, confirmedT)
 deaths_day1 = df_day1(deathsT, confirmedT)
 recovered_day1 = df_day1(recoveredT, confirmedT)
 mortality_day1 = df_day1(mortalityT, confirmedT)
 confirmednorm_day1 = confirmed_day1/confirmed_day1.loc[0,:]
maxday = confirmed_day1.shape[0]
head['elec_argmax'] = np.argmax(np.array(head[['noelec','coopele', 'public', 'planpri']]),
axis = 1 )<feature_engineering>
date_day1 = confirmedT.copy()
for column in date_day1:
date_day1[column] = confirmedT.index.tolist()
date_day1 = df_day1(date_day1, confirmedT )
head['tipovivi_argmax'] = np.argmax(np.array(head[['tipovivi5','tipovivi4', 'tipovivi3', 'tipovivi2','tipovivi1']]),
axis = 1 )<feature_engineering>
def logistic_curve(x, k, x_0, ymax):
return ymax /(1 + np.exp(-k*(x-x_0)) )
head['electro_house'] = head['refrig'] + head['computer'] + head['television']<feature_engineering>
def logistic_curve2(x, k1, k2, x_0, ymax):
return ymax /(1 + np.exp(-k1*(x-x_0)) + np.exp(-k2*(x-x_0)) )
head['diff_tamviv_hhsize'] = head['tamviv'] - head['hhsize']<drop_column>
list_countries = dftrain[dftrain['Date'] == '2020-01-22']['Country_Region'].tolist()
list_states = dftrain[dftrain['Date'] == '2020-01-22']['Province_State'].tolist()
datenow = datetime.now()
head.drop(labels = [ 'epared1', 'epared2', 'epared3','etecho1', 'etecho2', 'etecho3',
'eviv1', 'eviv2', 'eviv3',
'abastaguano', 'abastaguafuera','abastaguadentro',
'noelec','coopele', 'public', 'planpri',
'refrig', 'computer','television',
'tipovivi5','tipovivi4', 'tipovivi3', 'tipovivi2','tipovivi1',
'tamviv']
, axis = 1, inplace = True )<data_type_conversions>
list_date_pand = [] ; list_maxcases = []; list_maxfat = []
for country, state in list(zip(list_countries, list_states)) :
df2 = dftrain.loc[(dftrain['Country_Region'] == country)&(dftrain['Province_State'] == state)].fillna('None')
 maxcases, maxfat = df2['ConfirmedCases'].max() , df2['Fatalities'].max()
date_pand2 = []
date_pand = df2[df2['ConfirmedCases'] > 100.]['Date'].tolist()
try:
list_date_pand.append(pd.to_datetime(date_pand[0]))
except:
list_date_pand.append(pd.to_datetime(datenow))
list_maxcases.append(maxcases); list_maxfat.append(maxfat)
 dfstartpand = pd.DataFrame(np.array([list_countries, list_states, list_date_pand, list_maxcases, list_maxfat] ).T,
columns=['Country_Region', 'Province_State', 'Start_Pandemic', 'ConfirmedCases', 'Fatalities'])
 dfstartpand['Start_Pandemic'] = dfstartpand['Start_Pandemic'].dt.date
corr_matrix = indiv.corr()
upper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k=1 ).astype(np.bool))
to_drop = [column for column in upper.columns if any(abs(upper[column])> 0.95)]
to_drop<drop_column>
dfstartpand_ordered = dfstartpand.sort_values(by=['Start_Pandemic', 'ConfirmedCases', 'Fatalities'], ascending=[True, False, False])
 country_state_ordered = list(zip(dfstartpand_ordered['Country_Region'].tolist() , dfstartpand_ordered['Province_State']))
datetrain = dftrain['Date'].unique()
datetest = dftest['Date'].unique()
indiv.drop(labels = 'male', axis = 1, inplace = True )<feature_engineering>
dftest['ConfirmedCases_logreg'] = 0.0 ; dftrain['ConfirmedCases_logreg'] = 0.0
dftest['Fatalities_logreg'] = 0.0 ; dftrain['Fatalities_logreg'] = 0.0
p0 = 1
for country, state in country_state_ordered:
masktrain =(dftrain['Country_Region'] == country)&(dftrain['Province_State'] == state)
 masktrain2 =(dftrain['Country_Region'] == country)&(dftrain['Province_State'] == state)&(dftrain['Date'] <= '2020-03-31')&(dftrain['Date'] >= starttest)
 masktest =(dftest['Country_Region'] == country)&(dftest['Province_State'] == state)
 masktest2 =(dftest['Country_Region'] == country)&(dftest['Province_State'] == state)&(dftest['Date'] <= '2020-03-31')
 df2plot = dftrain[masktrain].set_index('Date')
 X = np.arange(len(df2plot))
X_test =(np.timedelta64(datetest[0]-datetrain[0], 'D')).astype(float)+np.arange(0,len(datetest))
try:
y = df2plot['ConfirmedCases']
p0_cases = [1/(len(X)/2.) , X[-1], y.max() ]
popt, pcov = curve_fit(logistic_curve, X, y, p0=p0_cases,bounds=([0,0,0],np.inf), maxfev=1000)
 k_cases, x_0_cases, ymax_cases = popt
cases_train_fc = pd.Series(logistic_curve(X, k_cases, x_0_cases, ymax_cases),index=df2plot.index)
 cases_test_fc = pd.Series(logistic_curve(X_test, k_cases, x_0_cases, ymax_cases),index=datetest)
 dftest.loc[masktest,'ConfirmedCases_logreg'] = cases_test_fc.tolist()
dftrain.loc[masktrain,'ConfirmedCases_logreg'] = cases_train_fc.tolist()
except:
print(country+' '+state+' Unable to fit the confirmed cases')
 dftest.loc[masktest,'ConfirmedCases_logreg'] = dftrain.loc[masktrain,'ConfirmedCases'].iloc[-1]
dftrain.loc[masktrain,'ConfirmedCases_logreg'] = dftrain.loc[masktrain,'ConfirmedCases']
try:
y = df2plot['Fatalities']
p0_deaths = [1/(len(X)/2.) , X[-1], y.max() ]
popt, pcov = curve_fit(logistic_curve, X, y, p0=p0_deaths,bounds=([0,0,0],np.inf), maxfev=1000)
 k_deaths, x_0_deaths, ymax_deaths = popt
deaths_train_fc = pd.Series(logistic_curve(X, k_deaths, x_0_deaths, ymax_deaths),index=datetrain)
 deaths_test_fc = pd.Series(logistic_curve(X_test, k_deaths, x_0_deaths, ymax_deaths),index=datetest)
 dftest.loc[masktest,'Fatalities_logreg'] = deaths_test_fc.tolist()
dftrain.loc[masktrain,'Fatalities_logreg'] = deaths_train_fc.tolist()
except:
print(country+' '+state+' Unable to fit the fatalities')
 dftest.loc[masktest,'Fatalities_logreg'] = dftrain.loc[masktrain,'Fatalities'].iloc[-1]
dftrain.loc[masktrain,'Fatalities_logreg'] = dftrain.loc[masktrain,'Fatalities']
dftest.loc[masktest2,'ConfirmedCases_logreg'] = dftrain.loc[masktrain2,'ConfirmedCases'].tolist()
dftest.loc[masktest2,'Fatalities_logreg'] = dftrain.loc[masktrain2,'Fatalities'].tolist()
indiv['instlevel_argmax'] = np.argmax(np.array(indiv[['instlevel1', 'instlevel2','instlevel3','instlevel4','instlevel5','instlevel6','instlevel7','instlevel8','instlevel9']]),
axis = 1 )<feature_engineering>
dfsubm['ConfirmedCases'] = dftest['ConfirmedCases_logreg']
dfsubm['Fatalities'] = dftest['Fatalities_logreg']
<drop_column><EOS> 
dfsubm.to_csv('submission.csv', index=False )