Kaggle competition: COVID19 Global Forecasting (Week 3)
test['y_hat_xgb_c'] = y_hat_xgb_c
test['y_hat_xgb_f'] = y_hat_xgb_f
test['y_hat_xgb_c2'] = y_hat_xgb_c2
print('Fixing Negative Predictions:'
, np.sum(test.y_hat_xgb_c < 0)
 , np.sum(test.y_hat_xgb_c2 < 0)
 , np.sum(test.y_hat_xgb_f< 0)
 , np.sum(test.wg< 0)
 , np.sum(test.ARIMA< 0)
 , np.sum(test.SARIMAX< 0)
 , np.sum(test.y_hat_log< 0))
test.loc[test.y_hat_xgb_c < 0, 'y_hat_xgb_c'] = 0
test.loc[test.y_hat_xgb_c2 < 0, 'y_hat_xgb_c2'] = 0
test.loc[test.y_hat_xgb_f < 0, 'y_hat_xgb_f'] = 0
test.loc[test.wg < 0, 'wg'] = 0
test.loc[test.ARIMA < 0, 'ARIMA'] = 0
test.loc[test.SARIMAX < 0, 'SARIMAX'] = 0
test.loc[test.y_hat_log < 0, 'y_hat_log'] = 0
print('done', datetime.now() )<count_missing_values>
plotly.offline.init_notebook_mode()
%matplotlib inline
def RMSLE(pred,actual):
return np.sqrt(np.mean(np.power(( np.log(pred+1)-np.log(actual+1)) ,2)))
print('Fixing Inf Predictions:'
, np.sum(test.y_hat_xgb_c.isnull())
 , np.sum(test.y_hat_xgb_c2.isnull())
 , np.sum(test.y_hat_xgb_f.isnull())
 , np.sum(test.wg.isnull())
 , np.sum(test.ARIMA.isnull())
 , np.sum(test.SARIMAX.isnull())
 , np.sum(test.y_hat_log.isnull()))
booll =(test['SARIMAX'].isnull())
 test.loc[booll, 'SARIMAX'] = test[booll]['y_hat_log']
booll =(test['ARIMA'].isnull())
 test.loc[booll, 'ARIMA'] = test[booll]['y_hat_log']
print('Fixing Inf Predictions:'
, np.sum(test.y_hat_xgb_c.isnull())
 , np.sum(test.y_hat_xgb_c2.isnull())
 , np.sum(test.y_hat_xgb_f.isnull())
 , np.sum(test.wg.isnull())
 , np.sum(test.ARIMA.isnull())
 , np.sum(test.SARIMAX.isnull())
 , np.sum(test.y_hat_log.isnull()))
print('done', datetime.now() )<filter>
pd.set_option('mode.chained_assignment', None)
 test = pd.read_csv(".. /input/covid19-global-forecasting-week-3/test.csv")
 train = pd.read_csv(".. /input/covid19-global-forecasting-week-3/train.csv")
 train['Province_State'].fillna('', inplace=True)
 test['Province_State'].fillna('', inplace=True)
 train['Date'] = pd.to_datetime(train['Date'])
 test['Date'] = pd.to_datetime(test['Date'])
 train = train.sort_values(['Country_Region','Province_State','Date'])
 test = test.sort_values(['Country_Region','Province_State','Date'] )
print('Fixing Unrealistic Predictions:',
np.sum(test['pop'] *(700/3100)< test.y_hat_xgb_c),
np.sum(test['pop'] *(700/3100)< test.y_hat_xgb_c2),
np.sum(test['pop'] *(700/3100)< test.y_hat_xgb_f),
np.sum(test['pop'] *(700/3100)< test.wg),
np.sum(test['pop'] *(700/3100)< test.ARIMA),
np.sum(test['pop'] *(700/3100)< test.SARIMAX),
np.sum(test['pop'] *(700/3100)< test.y_hat_log), 
)
 booll =(test['pop'] *(700/3100)< test.y_hat_xgb_c)
 test.loc[booll, 'y_hat_xgb_c'] = test[booll]['pop'] *(700/3100)
 booll =(test['pop'] *(700/3100)< test.y_hat_xgb_c2)
 test.loc[booll, 'y_hat_xgb_c2'] = test[booll]['pop'] *(700/3100)
 booll =(test['pop'] *(700/3100)< test.y_hat_xgb_f)
 test.loc[booll, 'y_hat_xgb_f'] = test[booll]['pop'] *(700/3100)
 booll =(test['pop'] *(700/3100)< test.wg)
 test.loc[booll, 'wg'] = test[booll]['pop'] *(700/3100)
 booll =(test['pop'] *(700/3100)< test.ARIMA)
 test.loc[booll, 'ARIMA'] = test[booll]['pop'] *(700/3100)
 booll =(test['pop'] *(700/3100)< test.SARIMAX)
 test.loc[booll, 'SARIMAX'] = test[booll]['pop'] *(700/3100)
 booll =(test['pop'] *(700/3100)< test.y_hat_log)
 test.loc[booll, 'y_hat_log'] = test[booll]['pop'] *(700/3100)
 print('Fixed Unrealistic Predictions:',
np.sum(test['pop'] *(700/3100)< test.y_hat_xgb_c),
np.sum(test['pop'] *(700/3100)< test.y_hat_xgb_f),
np.sum(test['pop'] *(700/3100)< test.wg),
np.sum(test['pop'] *(700/3100)< test.ARIMA),
np.sum(test['pop'] *(700/3100)< test.SARIMAX),
np.sum(test['pop'] *(700/3100)< test.y_hat_log), 
)
 print('done', datetime.now() )<feature_engineering>
FirstDate = train.groupby('Country_Region' ).min() ['Date'].unique() [0]
train['Last Confirm'] = train['ConfirmedCases'].shift(1)
 while train[(train['Last Confirm'] > train['ConfirmedCases'])&(train['Date'] > FirstDate)].shape[0] > 0:
train['Last Confirm'] = train['ConfirmedCases'].shift(1)
 train['Last Fatalities'] = train['Fatalities'].shift(1)
 train.loc[(train['Last Confirm'] > train['ConfirmedCases'])&(train['Date'] > FirstDate),'ConfirmedCases'] = train.loc[(train['Last Confirm'] > train['ConfirmedCases'])&(train['Date'] > FirstDate),'Last Confirm']
train.loc[(train['Last Fatalities'] > train['Fatalities'])&(train['Date'] > FirstDate),'Fatalities'] = train.loc[(train['Last Fatalities'] > train['Fatalities'])&(train['Date'] > FirstDate),'Last Fatalities']
train['Last Confirm'] = train['ConfirmedCases'].shift(1)
 train['Last Fatalities'] = train['Fatalities'].shift(1 )
test['y_hat_ens'] = test.y_hat_xgb_c *.15 + test.wg *.05 + test.y_hat_log *.30 + test['SARIMAX'] *.03 + test.y_hat_xgb_c2 *.47
test[(test.Province_State=='')&(test.Country_Region=='France')][['Date','y_hat_xgb_c','y_hat_xgb_c2', 'wg','SARIMAX', 'SARIMAX2','y_hat_log', 'y_hat_ens']]<count_missing_values>
feature_day = [1,20,50,100,200,500,1000]
def CreateInput(data):
feature = []
for day in feature_day:
data.loc[:,'Number day from ' + str(day)+ ' case'] = 0
if(train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['ConfirmedCases'] < day)]['Date'].count() > 0):
fromday = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['ConfirmedCases'] < day)]['Date'].max()
else:
fromday = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]['Date'].min()
for i in range(0, len(data)) :
if(data['Date'].iloc[i] > fromday):
day_denta = data['Date'].iloc[i] - fromday
data['Number day from ' + str(day)+ ' case'].iloc[i] = day_denta.days
feature = feature + ['Number day from ' + str(day)+ ' case']
return data[feature]
pred_data_all = pd.DataFrame()
with tqdm(total=len(train['Country_Region'].unique())) as pbar:
for country in train['Country_Region'].unique() :
for province in train[(train['Country_Region'] == country)]['Province_State'].unique() :
df_train = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]
df_test = test[(test['Country_Region'] == country)&(test['Province_State'] == province)]
X_train = CreateInput(df_train)
 y_train_confirmed = df_train['ConfirmedCases'].ravel()
y_train_fatalities = df_train['Fatalities'].ravel()
X_pred = CreateInput(df_test)
 feature_use = X_pred.columns[0]
for i in range(X_pred.shape[1] - 1,0,-1):
if(X_pred.iloc[0,i] > 0):
feature_use = X_pred.columns[i]
break
idx = X_train[X_train[feature_use] == 0].shape[0]
adjusted_X_train = X_train[idx:][feature_use].values.reshape(-1, 1)
 adjusted_y_train_confirmed = y_train_confirmed[idx:]
adjusted_y_train_fatalities = y_train_fatalities[idx:]
adjusted_X_pred = X_pred[feature_use].values.reshape(-1, 1)
 model = make_pipeline(PolynomialFeatures(2), BayesianRidge())
 model.fit(adjusted_X_train,adjusted_y_train_confirmed)
 y_hat_confirmed = model.predict(adjusted_X_pred)
 model.fit(adjusted_X_train,adjusted_y_train_fatalities)
 y_hat_fatalities = model.predict(adjusted_X_pred)
 pred_data = test[(test['Country_Region'] == country)&(test['Province_State'] == province)]
pred_data['ConfirmedCases_hat'] = y_hat_confirmed
pred_data['Fatalities_hat'] = y_hat_fatalities
pred_data_all = pred_data_all.append(pred_data)
 pbar.update(1)
 df_val = pd.merge(pred_data_all,train[['Date','Country_Region','Province_State','ConfirmedCases','Fatalities']],on=['Date','Country_Region','Province_State'], how='left')
 df_val.loc[df_val['Fatalities_hat'] < 0,'Fatalities_hat'] = 0
df_val.loc[df_val['ConfirmedCases_hat'] < 0,'ConfirmedCases_hat'] = 0
df_val_1 = df_val.copy()
RMSLE(df_val[(df_val['ConfirmedCases'].isnull() == False)]['ConfirmedCases'].values,df_val[(df_val['ConfirmedCases'].isnull() == False)]['ConfirmedCases_hat'].values )
print('Empty Predictions?', np.sum(test.y_hat_ens.isnull())) 
<compute_train_metric>
RMSLE(df_val[(df_val['Fatalities'].isnull() == False)]['Fatalities'].values,df_val[(df_val['Fatalities'].isnull() == False)]['Fatalities_hat'].values )
narf = train[train.day_from_jan_first== 79].copy()
narf = narf.merge(test[test.day_from_jan_first== 79], on=['Country_Region','Province_State','day_from_jan_first'])
 narf['err'] = np.abs(( narf.wg_y - narf.ConfirmedCases)/(1+narf.ConfirmedCases))
print(mean_squared_log_error(narf.ConfirmedCases,narf.y_hat_ens))
print('wg', mean_squared_log_error(narf.ConfirmedCases,narf.wg_y))
print('SARIMAX', mean_squared_log_error(narf.ConfirmedCases,narf.SARIMAX))
print('y_hat_log', mean_squared_log_error(narf.ConfirmedCases,narf.y_hat_log_y))
print('y_hat_xgb_c', mean_squared_log_error(narf.ConfirmedCases,narf.y_hat_xgb_c))
narf = narf[narf.err>100].copy()
if narf.shape[0]> 0:
print(mean_squared_log_error(narf.ConfirmedCases,narf.y_hat_ens))
else:
print('no fixes for wg_y')
 for index, row in narf.iterrows() :
country = row['Country_Region']
state = row['Province_State']
booll =(test.Country_Region==country)&(test.Province_State==state)
 test.loc[booll, 'wg'] =(test[booll].y_hat_log)*.3 +(test[booll].y_hat_xgb_c)*.7
print('done', datetime.now() )<feature_engineering>
val_score = []
for country in df_val['Country_Region'].unique() :
df_val_country = df_val[(df_val['Country_Region'] == country)&(df_val['Fatalities'].isnull() == False)]
val_score.append([country, RMSLE(df_val_country['ConfirmedCases'].values,df_val_country['ConfirmedCases_hat'].values),RMSLE(df_val_country['Fatalities'].values,df_val_country['Fatalities_hat'].values)])
 df_val_score = pd.DataFrame(val_score)
 df_val_score.columns = ['Country','ConfirmedCases_Scored','Fatalities_Scored']
df_val_score.sort_values('ConfirmedCases_Scored', ascending = False)
test['y_hat_ens'] = test.y_hat_xgb_c *.15 + test.wg *.05 + test.y_hat_log *.30 + test['SARIMAX'] *.03 + test.y_hat_xgb_c2 *.47
test[(test.Province_State=='')&(test.Country_Region=='France')][['Date','y_hat_xgb_c','y_hat_xgb_c2', 'wg', 'ARIMA','SARIMAX','y_hat_log', 'y_hat_ens']]<compute_train_metric>
country = "India"
df_val = df_val_1
df_val[df_val['Country_Region'] == country].groupby(['Date','Country_Region'] ).sum().reset_index()
narf = train[train.day_from_jan_first== 79].copy()
narf = narf.merge(test[test.day_from_jan_first== 79], on=['Country_Region','Province_State','day_from_jan_first'])
 narf['err'] = np.abs(( narf.y_hat_log_y - narf.ConfirmedCases)/(1+narf.ConfirmedCases))
print(mean_squared_log_error(narf.ConfirmedCases,narf.y_hat_ens))
print('wg', mean_squared_log_error(narf.ConfirmedCases,narf.wg_y))
print('SARIMAX', mean_squared_log_error(narf.ConfirmedCases,narf.SARIMAX))
print('y_hat_log', mean_squared_log_error(narf.ConfirmedCases,narf.y_hat_log_y))
print('y_hat_xgb_c', mean_squared_log_error(narf.ConfirmedCases,narf.y_hat_xgb_c))
narf = narf[narf.err>10].copy()
if narf.shape[0]> 0:
print(mean_squared_log_error(narf.ConfirmedCases,narf.y_hat_ens))
else:
print('no fixes for y_hat_log')
 for index, row in narf.iterrows() :
country = row['Country_Region']
state = row['Province_State']
booll =(test.Country_Region==country)&(test.Province_State==state)
 print('done', datetime.now() )<filter>
feature_day = [1,20,50,100,200,500,1000]
def CreateInput(data):
feature = []
for day in feature_day:
data.loc[:,'Number day from ' + str(day)+ ' case'] = 0
if(train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['ConfirmedCases'] < day)]['Date'].count() > 0):
fromday = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['ConfirmedCases'] < day)]['Date'].max()
else:
fromday = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]['Date'].min()
for i in range(0, len(data)) :
if(data['Date'].iloc[i] > fromday):
day_denta = data['Date'].iloc[i] - fromday
data['Number day from ' + str(day)+ ' case'].iloc[i] = day_denta.days
feature = feature + ['Number day from ' + str(day)+ ' case']
return data[feature]
pred_data_all = pd.DataFrame()
with tqdm(total=len(train['Country_Region'].unique())) as pbar:
for country in train['Country_Region'].unique() :
for province in train[(train['Country_Region'] == country)]['Province_State'].unique() :
with warnings.catch_warnings() :
warnings.filterwarnings("ignore")
 df_train = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]
df_test = test[(test['Country_Region'] == country)&(test['Province_State'] == province)]
X_train = CreateInput(df_train)
 y_train_confirmed = df_train['ConfirmedCases'].ravel()
y_train_fatalities = df_train['Fatalities'].ravel()
X_pred = CreateInput(df_test)
 feature_use = X_pred.columns[0]
for i in range(X_pred.shape[1] - 1,0,-1):
if(X_pred.iloc[0,i] > 0):
feature_use = X_pred.columns[i]
break
idx = X_train[X_train[feature_use] == 0].shape[0]
adjusted_X_train = X_train[idx:][feature_use].values.reshape(-1, 1)
 adjusted_y_train_confirmed = y_train_confirmed[idx:]
adjusted_y_train_fatalities = y_train_fatalities[idx:]
pred_data = test[(test['Country_Region'] == country)&(test['Province_State'] == province)]
max_train_date = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]['Date'].max()
min_test_date = pred_data['Date'].min()
model = ExponentialSmoothing(adjusted_y_train_confirmed, trend = 'additive' ).fit()
y_hat_confirmed = model.forecast(pred_data[pred_data['Date'] > max_train_date].shape[0])
 y_train_confirmed = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['Date'] >= min_test_date)]['ConfirmedCases'].values
y_hat_confirmed = np.concatenate(( y_train_confirmed,y_hat_confirmed), axis = 0)
 model = ExponentialSmoothing(adjusted_y_train_fatalities, trend = 'additive' ).fit()
y_hat_fatalities = model.forecast(pred_data[pred_data['Date'] > max_train_date].shape[0])
 y_train_fatalities = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['Date'] >= min_test_date)]['Fatalities'].values
y_hat_fatalities = np.concatenate(( y_train_fatalities,y_hat_fatalities), axis = 0)
 pred_data['ConfirmedCases_hat'] = y_hat_confirmed
pred_data['Fatalities_hat'] = y_hat_fatalities
pred_data_all = pred_data_all.append(pred_data)
 pbar.update(1)
 df_val = pd.merge(pred_data_all,train[['Date','Country_Region','Province_State','ConfirmedCases','Fatalities']],on=['Date','Country_Region','Province_State'], how='left')
 df_val.loc[df_val['Fatalities_hat'] < 0,'Fatalities_hat'] = 0
df_val.loc[df_val['ConfirmedCases_hat'] < 0,'ConfirmedCases_hat'] = 0
df_val_2 = df_val.copy()
test[test.y_hat_ens==1521225]
test[(test.Province_State=='')&(test.Country_Region=='Turkey')][['Date','y_hat_xgb_c', 'wg', 'ARIMA','SARIMAX','y_hat_log', 'y_hat_ens']]<data_type_conversions>
feature_day = [1,20,50,100,200,500,1000]
def CreateInput(data):
feature = []
for day in feature_day:
data.loc[:,'Number day from ' + str(day)+ ' case'] = 0
if(train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['ConfirmedCases'] < day)]['Date'].count() > 0):
fromday = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['ConfirmedCases'] < day)]['Date'].max()
else:
fromday = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]['Date'].min()
for i in range(0, len(data)) :
if(data['Date'].iloc[i] > fromday):
day_denta = data['Date'].iloc[i] - fromday
data['Number day from ' + str(day)+ ' case'].iloc[i] = day_denta.days
feature = feature + ['Number day from ' + str(day)+ ' case']
return data[feature]
pred_data_all = pd.DataFrame()
with tqdm(total=len(train['Country_Region'].unique())) as pbar:
for country in train['Country_Region'].unique() :
for province in train[(train['Country_Region'] == country)]['Province_State'].unique() :
with warnings.catch_warnings() :
warnings.filterwarnings("ignore")
 df_train = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]
df_test = test[(test['Country_Region'] == country)&(test['Province_State'] == province)]
X_train = CreateInput(df_train)
 y_train_confirmed = df_train['ConfirmedCases'].ravel()
y_train_fatalities = df_train['Fatalities'].ravel()
X_pred = CreateInput(df_test)
 feature_use = X_pred.columns[0]
for i in range(X_pred.shape[1] - 1,0,-1):
if(X_pred.iloc[0,i] > 0):
feature_use = X_pred.columns[i]
break
idx = X_train[X_train[feature_use] == 0].shape[0]
adjusted_X_train = X_train[idx:][feature_use].values.reshape(-1, 1)
 adjusted_y_train_confirmed = y_train_confirmed[idx:]
adjusted_y_train_fatalities = y_train_fatalities[idx:]
idx = X_pred[X_pred[feature_use] == 0].shape[0]
adjusted_X_pred = X_pred[idx:][feature_use].values.reshape(-1, 1)
 pred_data = test[(test['Country_Region'] == country)&(test['Province_State'] == province)]
max_train_date = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]['Date'].max()
min_test_date = pred_data['Date'].min()
model = SARIMAX(adjusted_y_train_confirmed, order=(1,1,0),
measurement_error=True ).fit(disp=False)
 y_hat_confirmed = model.forecast(pred_data[pred_data['Date'] > max_train_date].shape[0])
 y_train_confirmed = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['Date'] >= min_test_date)]['ConfirmedCases'].values
y_hat_confirmed = np.concatenate(( y_train_confirmed,y_hat_confirmed), axis = 0)
 model = SARIMAX(adjusted_y_train_fatalities, order=(1,1,0),
measurement_error=True ).fit(disp=False)
 y_hat_fatalities = model.forecast(pred_data[pred_data['Date'] > max_train_date].shape[0])
 y_train_fatalities = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['Date'] >= min_test_date)]['Fatalities'].values
y_hat_fatalities = np.concatenate(( y_train_fatalities,y_hat_fatalities), axis = 0)
 pred_data['ConfirmedCases_hat'] = y_hat_confirmed
pred_data['Fatalities_hat'] = y_hat_fatalities
pred_data_all = pred_data_all.append(pred_data)
 pbar.update(1)
 df_val = pd.merge(pred_data_all,train[['Date','Country_Region','Province_State','ConfirmedCases','Fatalities']],on=['Date','Country_Region','Province_State'], how='left')
 df_val.loc[df_val['Fatalities_hat'] < 0,'Fatalities_hat'] = 0
df_val.loc[df_val['ConfirmedCases_hat'] < 0,'ConfirmedCases_hat'] = 0
df_val_3 = df_val.copy()
test['y_hat_ens'] = test.y_hat_ens.astype(int)
 print(np.max(test['y_hat_ens'])) 
<save_to_csv>
method_list = ['Poly Bayesian Ridge','Exponential Smoothing','SARIMA']
method_val = [df_val_1,df_val_2,df_val_3]
for i in range(0,3):
df_val = method_val[i]
method_score = [method_list[i]] + [RMSLE(df_val[(df_val['ConfirmedCases'].isnull() == False)]['ConfirmedCases'].values,df_val[(df_val['ConfirmedCases'].isnull() == False)]['ConfirmedCases_hat'].values)] + [RMSLE(df_val[(df_val['Fatalities'].isnull() == False)]['Fatalities'].values,df_val[(df_val['Fatalities'].isnull() == False)]['Fatalities_hat'].values)]
print(method_score )
submissionOrig = pd.read_csv(".. /input/covid19-global-forecasting-week-2/submission.csv")
 submissionOrig["ConfirmedCases"]= pd.Series(test.y_hat_ens)
 submissionOrig["Fatalities"] = pd.Series(test.y_hat_xgb_f)
 submissionOrig.to_csv('submission.csv',index=False)
 submissionOrig.head(25)
 print('done', datetime.now() )<import_modules>
df_val = df_val_3
submission = df_val[['ForecastId','ConfirmedCases_hat','Fatalities_hat']]
submission.columns = ['ForecastId','ConfirmedCases','Fatalities']
submission.to_csv('submission.csv', index=False)
 submission