Kaggle competition: COVID19 Global Forecasting (Week 3)
poly_predicted_fatalities = pd.DataFrame()
for place in twodeg_places:
features = XYtrain[XYtrain['place']==place][['label','intercept']]
target = XYtrain[XYtrain['place']==place]['Fatalities']
Xtest = XYtest[XYtest['place']==place][['label','intercept']]
model = make_pipeline(PolynomialFeatures(2), Ridge())
 model.fit(np.array(features), target)
 y_pred = model.predict(np.array(Xtest))
a = pd.DataFrame(zip(XYtrain[XYtrain['place']==place]['place'],
y_pred.tolist()),columns=['place','Fatalities'])
 poly_predicted_fatalities = poly_predicted_fatalities.append(a)
 for place in threedeg_places:
features = XYtrain[XYtrain['place']==place][['label','intercept']]
target = XYtrain[XYtrain['place']==place]['Fatalities']
Xtest = XYtest[XYtest['place']==place][['label','intercept']]
model = make_pipeline(PolynomialFeatures(3), Ridge())
 model.fit(np.array(features), target)
 y_pred = model.predict(np.array(Xtest))
b = pd.DataFrame(zip(XYtrain[XYtrain['place']==place]['place'],
y_pred.tolist()),columns=['place','Fatalities'])
 poly_predicted_fatalities = poly_predicted_fatalities.append(b)
 for place in fourdeg_places:
features = XYtrain[XYtrain['place']==place][['label','intercept']]
target = XYtrain[XYtrain['place']==place]['Fatalities']
Xtest = XYtest[XYtest['place']==place][['label','intercept']]
model = make_pipeline(PolynomialFeatures(4), Ridge())
 model.fit(np.array(features), target)
 y_pred = model.predict(np.array(Xtest))
c = pd.DataFrame(zip(XYtrain[XYtrain['place']==place]['place'],
y_pred.tolist()),columns=['place','Fatalities'])
 poly_predicted_fatalities = poly_predicted_fatalities.append(c)
 for place in fivedeg_places:
features = XYtrain[XYtrain['place']==place][['label','intercept']]
target = XYtrain[XYtrain['place']==place]['Fatalities']
Xtest = XYtest[XYtest['place']==place][['label','intercept']]
model = make_pipeline(PolynomialFeatures(5), Ridge())
 model.fit(np.array(features), target)
 y_pred = model.predict(np.array(Xtest))
d = pd.DataFrame(zip(XYtrain[XYtrain['place']==place]['place'],
y_pred.tolist()),columns=['place','Fatalities'])
 poly_predicted_fatalities = poly_predicted_fatalities.append(d)
 for place in sevdeg_places:
features = XYtrain[XYtrain['place']==place][['label','intercept']]
target = XYtrain[XYtrain['place']==place]['Fatalities']
Xtest = XYtest[XYtest['place']==place][['label','intercept']]
model = make_pipeline(PolynomialFeatures(6), Ridge())
 model.fit(np.array(features), target)
 y_pred = model.predict(np.array(Xtest))
e = pd.DataFrame(zip(XYtrain[XYtrain['place']==place]['place'],
y_pred.tolist()),columns=['place','Fatalities'])
 poly_predicted_fatalities = poly_predicted_fatalities.append(e )<create_dataframe>
train_data = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-3/train.csv")
 test_data = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-3/test.csv" )
for place in nofit_places1:
e = poly_data[(poly_data['place']==place)&(poly_data['date']>'2020-03-18')]
f = e['ConfirmedCases'].fillna(method = 'ffill')
 g = pd.DataFrame(zip(e['place'], f),columns=['place','ConfirmedCases'])
 poly_predicted_confirmedcases = poly_predicted_confirmedcases.append(g)
 for place in nofit_places2:
h = poly_data[(poly_data['place']==place)&(poly_data['date']>'2020-03-18')]
i = h['Fatalities'].fillna(method = 'ffill')
 j = pd.DataFrame(zip(h['place'], i),columns=['place','Fatalities'])
 poly_predicted_fatalities = poly_predicted_fatalities.append(j )<prepare_output>
import matplotlib.pyplot as plt
import seaborn as sns
import plotly.express as px
poly_predicted_confirmedcases2= pd.DataFrame({'date':XYtest.date,
'place':poly_predicted_confirmedcases['place'].tolist() ,
'ConfirmedCases':poly_predicted_confirmedcases['ConfirmedCases'].tolist() })
 poly_predicted_confirmedcases2.head()<prepare_output>
train =train_data.drop(["Province_State"] , axis = 1)
 test =test_data.drop(["Province_State"] , axis = 1)
poly_predicted_fatalities2= pd.DataFrame({'date':XYtest.date,
'place':poly_predicted_fatalities['place'].tolist() ,
'Fatalities':poly_predicted_fatalities['Fatalities'].tolist() })
 poly_predicted_fatalities2.head()<merge>
train = train[train["ConfirmedCases"] != 0]
poly_compiled = poly_predicted_confirmedcases2.merge(poly_predicted_fatalities2, how='inner', on=['place','date'] )<merge>
df = train.fillna('NA' ).groupby(['Country_Region','Date'])['ConfirmedCases'].sum() \ 
.groupby(['Country_Region'] ).max().sort_values() \ 
.groupby(['Country_Region'] ).sum().sort_values(ascending = False)
 top10 = pd.DataFrame(df ).head(10 ).reset_index()
top10
test_poly_compiled= test.merge(poly_compiled, how='inner', on=['place','date'])
 test_poly_compiled= test_poly_compiled.set_index('date')
 test_poly_compiled<prepare_output>
deaths = train.fillna('NA' ).groupby(['Country_Region','Date'])['Fatalities'].sum() \ 
.groupby(['Country_Region'] ).max().sort_values() \ 
.groupby(['Country_Region'] ).sum().sort_values(ascending = False)
 deaths10 = pd.DataFrame(deaths ).head(10 ).reset_index()
deaths10
df_compiled = pd.DataFrame()
for place in test_poly_compiled.place.unique() :
a = test_poly_compiled[test_poly_compiled['place']==place]
ind_max_confirmedcases = np.argmax(a['ConfirmedCases'])
 a = a.replace(to_replace=a.loc[(a.index>ind_max_confirmedcases),'ConfirmedCases'].tolist() ,
value=a.loc[ind_max_confirmedcases,'ConfirmedCases'])
 ind_max_fatatities = np.argmax(a['Fatalities'])
 a = a.replace(to_replace=a.loc[(a.index>ind_max_fatatities),'Fatalities'].tolist() ,
value=a.loc[ind_max_fatatities,'Fatalities'])
 df_compiled = df_compiled.append(a)
 df_compiled[df_compiled['place']=='_Zimbabwe'].tail()<load_from_csv>
data_by_date = pd.DataFrame(train.groupby(["Country_Region" , "Date"])["ConfirmedCases"].sum())
 data_by_date = data_by_date.sort_values("ConfirmedCases" , axis=0 , ascending = True ).reset_index()
submission= pd.read_csv(datapath+'submission.csv' )<merge>
death_by_date = pd.DataFrame(train.groupby(["Country_Region" , "Date"])["Fatalities"].sum())
 death_by_date = death_by_date.sort_values("Fatalities" , axis=0 , ascending = False ).reset_index()
sub2 = submission[['ForecastId']].merge(df_compiled[['ForecastId','ConfirmedCases','Fatalities']],
how='left',on='ForecastId' )<feature_engineering>
df = data_by_date.loc[(data_by_date['Country_Region'] == 'US')&(data_by_date.Date >= '2020-03-01')]
df = df.sort_values('ConfirmedCases',ascending = True )
sub2['ConfirmedCases'] = sub2['ConfirmedCases'].round(1)
 sub2['Fatalities'] = sub2['Fatalities'].round(1 ).abs()<save_to_csv>
death_US = death_by_date.loc[(death_by_date['Country_Region'] == 'US')&(death_by_date.Date >= '2020-03-01')]
death_US = death_US.sort_values('Fatalities',ascending = True )
sub2.to_csv('submission.csv', index=False )<compute_test_metric>
data_china = data_by_date.loc[(data_by_date["Country_Region"] == "China")&(data_by_date["Date"] >= "2020-01-01")]
def RMSLE(pred,actual):
return np.sqrt(np.mean(np.power(( np.log(pred+1)-np.log(actual+1)) ,2)) )<load_from_csv>
death_china = death_by_date.loc[(death_by_date["Country_Region"] == "China")&(death_by_date["Date"] >= "2020-01-01")]
death_china.sort_values("Fatalities" , ascending = True )
pd.set_option('mode.chained_assignment', None)
 test = pd.read_csv(".. /input/covid19-global-forecasting-week-2/test.csv")
 train = pd.read_csv(".. /input/covid19-global-forecasting-week-2/train.csv")
 train['Province_State'].fillna('', inplace=True)
 test['Province_State'].fillna('', inplace=True)
 train['Date'] = pd.to_datetime(train['Date'])
 test['Date'] = pd.to_datetime(test['Date'])
 train = train.sort_values(['Country_Region','Province_State','Date'])
 test = test.sort_values(['Country_Region','Province_State','Date'] )<groupby>
xtrain = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-3/train.csv')
 xtest = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-3/test.csv')
 xsubmission = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-3/submission.csv' )
train[['ConfirmedCases', 'Fatalities']] = train.groupby(['Country_Region', 'Province_State'])[['ConfirmedCases', 'Fatalities']].transform('cummax' )<feature_engineering>
xtrain.rename(columns={'Country_Region':'Country'}, inplace=True)
 xtest.rename(columns={'Country_Region':'Country'}, inplace=True)
 xtrain.rename(columns={'Province_State':'State'}, inplace=True)
 xtest.rename(columns={'Province_State':'State'}, inplace=True)
 xtrain['Date'] = pd.to_datetime(xtrain['Date'], infer_datetime_format=True)
 xtest['Date'] = pd.to_datetime(xtest['Date'], infer_datetime_format=True)
 xtrain.info()
xtest.info()
y1_xTrain = xtrain.iloc[:, -2]
y1_xTrain.head()
y2_xTrain = xtrain.iloc[:, -1]
y2_xTrain.head()
EMPTY_VAL = "EMPTY_VAL"
def fillState(state, country):
if state == EMPTY_VAL: return country
return state
feature_day = [1,20,50,100,200,500,1000]
def CreateInput(data):
feature = []
for day in feature_day:
data.loc[:,'Number day from ' + str(day)+ ' case'] = 0
if(train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['ConfirmedCases'] < day)]['Date'].count() > 0):
fromday = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['ConfirmedCases'] < day)]['Date'].max()
else:
fromday = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]['Date'].min()
for i in range(0, len(data)) :
if(data['Date'].iloc[i] > fromday):
day_denta = data['Date'].iloc[i] - fromday
data['Number day from ' + str(day)+ ' case'].iloc[i] = day_denta.days
feature = feature + ['Number day from ' + str(day)+ ' case']
return data[feature]
pred_data_all = pd.DataFrame()
with tqdm(total=len(train['Country_Region'].unique())) as pbar:
for country in train['Country_Region'].unique() :
for province in train[(train['Country_Region'] == country)]['Province_State'].unique() :
df_train = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]
df_test = test[(test['Country_Region'] == country)&(test['Province_State'] == province)]
X_train = CreateInput(df_train)
 y_train_confirmed = df_train['ConfirmedCases'].ravel()
y_train_fatalities = df_train['Fatalities'].ravel()
X_pred = CreateInput(df_test)
 feature_use = X_pred.columns[0]
for i in range(X_pred.shape[1] - 1,0,-1):
if(X_pred.iloc[0,i] > 0):
feature_use = X_pred.columns[i]
break
idx = X_train[X_train[feature_use] == 0].shape[0]
adjusted_X_train = X_train[idx:][feature_use].values.reshape(-1, 1)
 adjusted_y_train_confirmed = y_train_confirmed[idx:]
adjusted_y_train_fatalities = y_train_fatalities[idx:]
adjusted_X_pred = X_pred[feature_use].values.reshape(-1, 1)
 model = make_pipeline(PolynomialFeatures(2), BayesianRidge())
 model.fit(adjusted_X_train,adjusted_y_train_confirmed)
 y_hat_confirmed = model.predict(adjusted_X_pred)
 model.fit(adjusted_X_train,adjusted_y_train_fatalities)
 y_hat_fatalities = model.predict(adjusted_X_pred)
 pred_data = test[(test['Country_Region'] == country)&(test['Province_State'] == province)]
pred_data['ConfirmedCases_hat'] = y_hat_confirmed
pred_data['Fatalities_hat'] = y_hat_fatalities
pred_data_all = pred_data_all.append(pred_data)
 pbar.update(1)
 df_val = pd.merge(pred_data_all,train[['Date','Country_Region','Province_State','ConfirmedCases','Fatalities']],on=['Date','Country_Region','Province_State'], how='left')
 df_val.loc[df_val['Fatalities_hat'] < 0,'Fatalities_hat'] = 0
df_val.loc[df_val['ConfirmedCases_hat'] < 0,'ConfirmedCases_hat'] = 0
df_val_1 = df_val.copy()<compute_test_metric>
X_xTrain = xtrain.copy()
RMSLE(df_val[(df_val['ConfirmedCases'].isnull() == False)]['ConfirmedCases'].values,df_val[(df_val['ConfirmedCases'].isnull() == False)]['ConfirmedCases_hat'].values )<compute_test_metric>
X_xTrain['State'].fillna(EMPTY_VAL, inplace=True)
 X_xTrain['State'] = X_xTrain.loc[:, ['State', 'Country']].apply(lambda x : fillState(x['State'], x['Country']), axis=1)
 X_xTrain.loc[:, 'Date'] = X_xTrain.Date.dt.strftime("%m%d")
 X_xTrain["Date"] = X_xTrain["Date"].astype(int)
 X_xTrain.head()
X_xTest = xtest.copy()
X_xTest['State'].fillna(EMPTY_VAL, inplace=True)
 X_xTest['State'] = X_xTest.loc[:, ['State', 'Country']].apply(lambda x : fillState(x['State'], x['Country']), axis=1)
 X_xTest.loc[:, 'Date'] = X_xTest.Date.dt.strftime("%m%d")
 X_xTest["Date"] = X_xTest["Date"].astype(int)
 X_xTest.head()
RMSLE(df_val[(df_val['Fatalities'].isnull() == False)]['Fatalities'].values,df_val[(df_val['Fatalities'].isnull() == False)]['Fatalities_hat'].values )<compute_test_metric>
le = preprocessing.LabelEncoder()
X_xTrain.Country = le.fit_transform(X_xTrain.Country)
 X_xTrain['State'] = le.fit_transform(X_xTrain['State'])
 X_xTrain.head()
X_xTest.Country = le.fit_transform(X_xTest.Country)
 X_xTest['State'] = le.fit_transform(X_xTest['State'])
 X_xTest.head()
xtrain.head()
xtrain.loc[xtrain.Country == 'Afghanistan', :]
xtest.tail()
method_list = ['Poly Bayesian Ridge','SARIMA']
method_val = [df_val_1,df_val_3]
for i in range(0,2):
df_val = method_val[i]
method_score = [method_list[i]] + [RMSLE(df_val[(df_val['ConfirmedCases'].isnull() == False)]['ConfirmedCases'].values,df_val[(df_val['ConfirmedCases'].isnull() == False)]['ConfirmedCases_hat'].values)] + [RMSLE(df_val[(df_val['Fatalities'].isnull() == False)]['Fatalities'].values,df_val[(df_val['Fatalities'].isnull() == False)]['Fatalities_hat'].values)]
print(method_score )<save_to_csv>
filterwarnings('ignore')
 le = preprocessing.LabelEncoder()
countries = X_xTrain.Country.unique()
df_val = df_val_1
submission = df_val[['ForecastId','ConfirmedCases_hat','Fatalities_hat']]
submission.columns = ['ForecastId','ConfirmedCases','Fatalities']
submission.to_csv('submission.csv', index=False)
 submission<set_options>
xout = pd.DataFrame({'ForecastId': [], 'ConfirmedCases': [], 'Fatalities': []} )
pd.set_option('display.max_columns',500)
 filterwarnings('ignore' )<load_from_csv>
for country in countries:
states = X_xTrain.loc[X_xTrain.Country == country, :].State.unique()
for state in states:
X_xTrain_CS = X_xTrain.loc[(X_xTrain.Country == country)&(X_xTrain.State == state), ['State', 'Country', 'Date', 'ConfirmedCases', 'Fatalities']]
y1_xTrain_CS = X_xTrain_CS.loc[:, 'ConfirmedCases']
y2_xTrain_CS = X_xTrain_CS.loc[:, 'Fatalities']
X_xTrain_CS = X_xTrain_CS.loc[:, ['State', 'Country', 'Date']]
X_xTrain_CS.Country = le.fit_transform(X_xTrain_CS.Country)
 X_xTrain_CS['State'] = le.fit_transform(X_xTrain_CS['State'])
 X_xTest_CS = X_xTest.loc[(X_xTest.Country == country)&(X_xTest.State == state), ['State', 'Country', 'Date', 'ForecastId']]
X_xTest_CS_Id = X_xTest_CS.loc[:, 'ForecastId']
X_xTest_CS = X_xTest_CS.loc[:, ['State', 'Country', 'Date']]
X_xTest_CS.Country = le.fit_transform(X_xTest_CS.Country)
 X_xTest_CS['State'] = le.fit_transform(X_xTest_CS['State'])
 xmodel1 = XGBRegressor(n_estimators=1000)
 xmodel1.fit(X_xTrain_CS, y1_xTrain_CS)
 y1_xpred = xmodel1.predict(X_xTest_CS)
 xmodel2 = XGBRegressor(n_estimators=1000)
 xmodel2.fit(X_xTrain_CS, y2_xTrain_CS)
 y2_xpred = xmodel2.predict(X_xTest_CS)
 xdata = pd.DataFrame({'ForecastId': X_xTest_CS_Id, 'ConfirmedCases': y1_xpred, 'Fatalities': y2_xpred})
 xout = pd.concat([xout, xdata], axis=0 )
train = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-2/train.csv', parse_dates=['Date'])
 test = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-2/test.csv', parse_dates=['Date'] )<rename_columns>
xout.ForecastId = xout.ForecastId.astype('int')
 xout.tail()
xout.to_csv('submission.csv', index=False )