Kaggle competition: COVID19 Global Forecasting (Week 4)
<SOS> metric: MCRMSLE Kaggle data source: covid19-global-forecasting-(week-4)<data_type_conversions>
py.init_notebook_mode(connected=True)
 pio.templates.default = "plotly_dark"
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
for item in('GarageYrBlt', 'GarageArea', 'GarageCars','BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF','TotalBsmtSF', 'BsmtFullBath', 'BsmtHalfBath'):
df[item] = df[item].fillna(0 )<feature_engineering>
train_df = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/train.csv')
 test_df = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/test.csv')
 submission_df = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/submission.csv')
 train_df = train_df.drop(['Id'],axis=1)
 train_df.rename(columns={'Country_Region':'Country'}, inplace=True)
 test_df.rename(columns={'Country_Region':'Country'}, inplace=True)
 train_df['Province_State'].fillna("None*", inplace = True)
 test_df['Province_State'].fillna("None*", inplace = True)
 renameCountryNames = {
"Congo(Brazzaville)": "Congo1",
"Congo(Kinshasa)": "Congo2",
"Cote d'Ivoire": "Côte d'Ivoire",
"Czechia": "Czech Republic(Czechia)",
"Korea, South": "South Korea",
"Saint Kitts and Nevis": "Saint Kitts & Nevis",
"Saint Vincent and the Grenadines": "St.Vincent & Grenadines",
"Taiwan*": "Taiwan",
"US": "United States"
}
train_df.replace({'Country': renameCountryNames}, inplace=True)
 test_df.replace({'Country': renameCountryNames}, inplace=True)
 specific_countries = ['United States', 'United Kingdom','Netherlands']
days_df = train_df['Date'].apply(lambda dt: datetime.datetime.strptime(dt, '%Y-%m-%d')- datetime.datetime.strptime('2020-01-21', '%Y-%m-%d')).apply(lambda x : str(x ).split() [0] ).astype(int)
 train_df['Days'] = days_df
days_df = test_df['Date'].apply(lambda dt: datetime.datetime.strptime(dt, '%Y-%m-%d')- datetime.datetime.strptime('2020-01-21', '%Y-%m-%d')).apply(lambda x : str(x ).split() [0] ).astype(int)
 test_df['Days'] = days_df
all_countries = train_df['Country'].unique()
display(train_df.tail())
 display(test_df.head())
 ww_df = train_df.groupby('Date')[['ConfirmedCases', 'Fatalities']].sum().reset_index()
ww_df['new_case'] = ww_df['ConfirmedCases'] - ww_df['ConfirmedCases'].shift(1)
 ww_df['new_deaths'] = ww_df['Fatalities'] - ww_df['Fatalities'].shift(1)
 country_df = train_df.groupby(['Date', 'Country'])[['ConfirmedCases', 'Fatalities']].sum().reset_index()
target_date = country_df['Date'].max()
train_end_day = train_df['Days'].max()
test_start_day = test_df['Days'].min()
test_end_day = test_df['Days'].max()
display(country_df[country_df['Country']=='France'][80:] )
df["MasVnrType"] = df["MasVnrType"].fillna("None")
 df["MasVnrArea"] = df["MasVnrArea"].fillna(0)
 df['MSZoning'] = df['MSZoning'].fillna(df['MSZoning'].mode() [0])
<count_values>
country_province_df = train_df[train_df['Country']=='United States'].groupby(['Date', 'Province_State'])[['ConfirmedCases', 'Fatalities']].sum().reset_index()
top_province_df = country_province_df.query('(Date == @target_date)' ).sort_values('ConfirmedCases', ascending=False)
 top30_provinces = top_province_df.sort_values('ConfirmedCases', ascending=False ).iloc[:30]['Province_State'].unique()
country_province_df['prev_cases'] = country_province_df.groupby('Province_State')['ConfirmedCases'].shift(1)
 country_province_df['New Case'] = country_province_df['ConfirmedCases'] - country_province_df['prev_cases']
country_province_df['New Case'].fillna(0, inplace=True)
 country_province_df['prev_deaths'] = country_province_df.groupby('Province_State')['Fatalities'].shift(1)
 country_province_df['New Death'] = country_province_df['Fatalities'] - country_province_df['prev_deaths']
country_province_df['New Death'].fillna(0, inplace=True)
 display(country_province_df[-30:])
 for province in top30_provinces:
present_country_df = country_province_df[country_province_df['Province_State']==province]
px.bar(present_country_df,
x='Date', y='New Case', color='Province_State',
title=f'United States : DAILY NEW Confirmed cases in '+province ).show()
df["Utilities"].value_counts()<drop_column>
def get_time_series(df,country_name,insert=False):
if df[df['Country'] == country_name]['Province_State'].nunique() > 1:
country_table = df[df['Country'] == country_name]
if insert:
country_df = pd.DataFrame(pd.pivot_table(country_table, values = ['ConfirmedCases','Fatalities','Days'],
index='Date', aggfunc=sum ).to_records())
 return country_df.set_index('Date')[['ConfirmedCases','Fatalities']]
return country_table.set_index('Date')[['Province_State','ConfirmedCases','Fatalities','Days']]
df = df[(df['Country'] == country_name)]
return df.set_index('Date')[['ConfirmedCases','Fatalities','Days']]
def get_time_series_province(province):
df = full_table[(full_table['Province_State'] == province)]
return df.set_index('Date')[['ConfirmedCases','Fatalities']]
province_country_dfs = {}
no_province_country_dfs = {}
absent_country_in_age_data_dfs = {}
province_country_dfs_list=[]
no_province_country_dfs_list=[]
absent_country_in_age_data_dfs_list=[]
province_countries=train_df[train_df['Province_State']!="None*"]['Country'].unique()
no_province_countries=train_df[train_df['Province_State']=="None*"]['Country'].unique()
no_province_countries= [x for x in no_province_countries if x not in province_countries]
for country in province_countries:
province_country_dfs[country] = get_time_series(train_df,country)
 for country in no_province_countries:
no_province_country_dfs[country] = get_time_series(train_df,country)
 print([x for x in no_province_countries if x in top30_countries])
 print([x for x in province_countries if x in top30_countries])
 assert(len([x for x in all_countries if x not in list(no_province_countries)+list(province_countries)])==0)
 display(province_country_dfs['United States'] )
df = df.drop(['Utilities'], axis=1 )<count_values>
print(tf.__version__ )
df["Functional"].value_counts()<feature_engineering>
train_df['Province_State'].fillna("None*", inplace = True)
 test_df['Province_State'].fillna("None*", inplace = True)
 test_df["ConfirmedCases"] = np.nan
test_df["Fatalities"] = np.nan
test_df = test_df.set_index('Date')
 display(test_df)
 prediction={}
df["Functional"] = df["Functional"].fillna("Typ")
<feature_engineering>
def build_model(n_steps):
model = Sequential()
model.add(LSTM(50, activation='relu', return_sequences=True, input_shape=(n_steps, 1)))
 model.add(LSTM(50, activation='relu'))
model.add(Dense(1))
model.compile(optimizer='RMSprop', loss='mse')
 return model
def split_sequence(sequence, n_steps):
	X, y = list() , list()
	for i in range(len(sequence)) :
		
		end_ix = i + n_steps
		
		if end_ix > len(sequence)-1:
			break
		
		seq_x, seq_y = sequence[i:end_ix], sequence[end_ix]
		X.append(seq_x)
 		y.append(seq_y)
 	return array(X), array(y )
print(df["Electrical"].mode())
 df['Electrical'] = df['Electrical'].fillna(df['Electrical'].mode() [0] )<feature_engineering>
test_country_df = test_df.groupby(['Date', 'Country'])[['ConfirmedCases', 'Fatalities']].sum().reset_index()
display(test_country_df[test_country_df['Country']=='Australia'][:20])
 for country in [x for x in province_countries if x in top30_countries]:
present_country_df = test_country_df[test_country_df['Country']==country].reset_index()
present_country_df['prev_cases'] = present_country_df.groupby('Country')['ConfirmedCases'].shift(1)
 present_country_df['New Case'] = present_country_df['ConfirmedCases'] - present_country_df['prev_cases']
present_country_df['New Case'].fillna(0, inplace=True)
 present_country_df['prev_deaths'] = present_country_df.groupby('Country')['Fatalities'].shift(1)
 present_country_df['New Death'] = present_country_df['Fatalities'] - present_country_df['prev_deaths']
present_country_df['New Death'].fillna(0, inplace=True)
 px.bar(present_country_df,
x='Date', y='New Case', color='Country',
title=f'DAILY NEW Confirmed cases in '+country ).show()
print(df["KitchenQual"].mode())
 df['KitchenQual'] = df['KitchenQual'].fillna(df['KitchenQual'].mode() [0])
 df['Exterior1st'] = df['Exterior1st'].fillna(df['Exterior1st'].mode() [0])
 df['Exterior2nd'] = df['Exterior2nd'].fillna(df['Exterior2nd'].mode() [0])
 df['SaleType'] = df['SaleType'].fillna(df['SaleType'].mode() [0] )<create_dataframe>
for country in no_province_countries:
current_considered_country_df = no_province_country_dfs[country][['ConfirmedCases','Fatalities','Days']].reset_index()
print(country)
 for i in range(train_end_day-test_start_day+1):
test_df.loc[(test_df['Country']==country)&(test_df['Days']==i+test_start_day), 'ConfirmedCases'] = current_considered_country_df.loc[current_considered_country_df['Days'] == i+test_start_day, 'ConfirmedCases'].values[0]
test_df.loc[(test_df['Country']==country)&(test_df['Days']==i+test_start_day), 'Fatalities'] = current_considered_country_df.loc[current_considered_country_df['Days'] == i+test_start_day, 'Fatalities'].values[0]
indexNames = current_considered_country_df[ current_considered_country_df['ConfirmedCases'] == 0 ].index
current_considered_country_df.drop(indexNames , inplace=True)
 cases_train = np.diff(current_considered_country_df['ConfirmedCases'].to_numpy())
 fatalities_train = np.diff(current_considered_country_df['Fatalities'].to_numpy())
 fatal_rate = 0.0
if(current_considered_country_df['Fatalities'].to_numpy() [-1]>0):
fatal_rate = current_considered_country_df['Fatalities'].to_numpy() [-1]/current_considered_country_df['ConfirmedCases'].to_numpy() [-1]
print("fatal rate is: "+str(fatal_rate))
cases_increase_avg = 0
days=0
for i in range(len(cases_train)-1):
cases_increase_avg +=(cases_train[i+1] -cases_train[i])
 days+=1
if(days>0):
cases_increase_avg = int(cases_increase_avg/days)
 fatal_increase_avg = 0
days=0
for i in range(len(fatalities_train)-1):
fatal_increase_avg +=(fatalities_train[i+1] -fatalities_train[i])
 days+=1
if(days>0):
fatal_increase_avg = int(fatal_increase_avg/days)
 n_steps = max(int(len(cases_train)*0.15),2)
 avg_weekly_per_day_case = []
avg_window = 4
avg_step = 2
X_cases_val, y_cases_val = split_sequence(cases_train,n_steps)
 X_cases_val = np.reshape(X_cases_val,(X_cases_val.shape[0],X_cases_val.shape[1],1))
model_cases=build_model(n_steps)
 model_cases.fit(X_cases_val, y_cases_val, epochs=50, verbose=0)
 cases_predict_next = model_cases.predict(np.reshape(X_cases_val[len(X_cases_val)-1],(1,n_steps,1)) , verbose=0 ).astype(int)
 cases_predict_next[0] = np.array([max(0,cases_predict_next[0])])
 X_fatal_val, y_fatal_val = split_sequence(fatalities_train,n_steps)
 X_fatal_val = np.reshape(X_fatal_val,(X_fatal_val.shape[0],X_fatal_val.shape[1],1))
model_fatalities=build_model(n_steps)
 model_fatalities.fit(X_fatal_val, y_fatal_val, epochs=50, verbose=0)
 fatality_predict_next = model_fatalities.predict(np.reshape(X_fatal_val[len(X_fatal_val)-1],(1,n_steps,1)) , verbose=0 ).astype(int)
 fatality_predict_next[0] = np.array([max(0,fatality_predict_next[0])])
 fatality_predict_next[0] = np.array([max(fatality_predict_next[0],cases_predict_next[0]*fatal_rate)])
 test_df.loc[(test_df['Country']==country)&(test_df['Days']==train_end_day+1), 'ConfirmedCases'] = test_df.loc[(test_df['Country']==country)&(test_df['Days']==train_end_day), 'ConfirmedCases'].values[0] + cases_predict_next[0]
test_df.loc[(test_df['Country']==country)&(test_df['Days']==train_end_day+1), 'Fatalities'] = test_df.loc[(test_df['Country']==country)&(test_df['Days']==train_end_day), 'Fatalities'].values[0] + fatality_predict_next[0]
new_entry_cases=X_cases_val[len(X_cases_val)-1]
new_entry_fatal=X_fatal_val[len(X_fatal_val)-1]
for i in range(test_end_day-train_end_day-1):
last_series = np.reshape(new_entry_cases,(1,n_steps,1))
new_entry_cases=np.delete(last_series,[0])
 new_entry_cases = np.insert(new_entry_cases,n_steps-1,cases_predict_next[0])
 cases_predict_next = model_cases.predict(np.reshape(new_entry_cases,(1,n_steps,1)) , verbose=0 ).astype(int)
 if(cases_predict_next[0]-new_entry_cases[n_steps-1]>cases_increase_avg):
cases_predict_next[0] = np.array([max(0,new_entry_cases[n_steps-1]+cases_increase_avg)])
 cases_predict_next[0] = np.array([max(0,cases_predict_next[0])])
 last_series = np.reshape(new_entry_fatal,(1,n_steps,1))
new_entry_fatal=np.delete(last_series,[0])
 new_entry_fatal = np.insert(new_entry_fatal,n_steps-1,fatality_predict_next[0])
 fatality_predict_next = model_fatalities.predict(np.reshape(new_entry_fatal,(1,n_steps,1)) , verbose=0 ).astype(int)
 if(fatality_predict_next[0]-new_entry_fatal[n_steps-1]>fatal_increase_avg):
fatality_predict_next[0] = max(0,new_entry_fatal[n_steps-1]+fatal_increase_avg)
 fatality_predict_next[0] = np.array([max(0,fatality_predict_next[0])])
 fatality_predict_next[0] = np.array([max(fatality_predict_next[0],int(cases_predict_next[0]*fatal_rate)) ])
 test_df.loc[(test_df['Country']==country)&(test_df['Days']==i+train_end_day+2), 'ConfirmedCases'] = test_df.loc[(test_df['Country']==country)&(test_df['Days']==i+train_end_day+1), 'ConfirmedCases'].values[0] + cases_predict_next[0]
test_df.loc[(test_df['Country']==country)&(test_df['Days']==i+train_end_day+2), 'Fatalities'] = test_df.loc[(test_df['Country']==country)&(test_df['Days']==i+train_end_day+1), 'Fatalities'].values[0] + fatality_predict_next[0]
del model_fatalities
del model_cases
missing_vl_ratio = df.isna().mean().round(4)* 100
missing_vl_ratio = missing_vl_ratio.drop(missing_vl_ratio[missing_vl_ratio == 0].index ).sort_values(ascending=False)[:30]
dataframe_miss=pd.DataFrame(missing_vl_ratio, columns=['missing ratio'])
 dataframe_miss.head(80 )<count_values>
test_df_copy = test_df
submission_df_copy = submission_df
submit = pd.DataFrame()
submit['ForecastId'] = test_df['ForecastId']
submit['ConfirmedCases'] = test_df['ConfirmedCases']
submit['Fatalities'] = test_df['Fatalities']
submit = submit.reset_index()
submit = submit.drop(['Date'], axis=1)
 display(submit.tail())
 submit.to_csv('submission.csv',index=False )