Kaggle competition: COVID19 Global Forecasting (Week 3)
<SOS> metric: MCRMSLE Kaggle data source: covid19-global-forecasting-week-3<compute_test_metric>
plotly.offline.init_notebook_mode()
%matplotlib inline
def RMSLE(pred,actual):
return np.sqrt(np.mean(np.power(( np.log(pred+1)-np.log(actual+1)) ,2)) )
RMSLE(df_val[(df_val['Fatalities'].isnull() == False)]['Fatalities'].values,df_val[(df_val['Fatalities'].isnull() == False)]['Fatalities_hat'].values )<groupby>
warnings.filterwarnings("ignore")
 %matplotlib inline
%config InlineBackend.figure_format = 'retina'
df_val[df_val['Country_Region'] == country].groupby(['Date','Country_Region'] ).sum().reset_index()<feature_engineering>
train = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-3/train.csv")
 test = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-3/test.csv")
 region_metadata = pd.read_csv("/kaggle/input/covid19-forecasting-metadata/region_metadata.csv")
 region_date_metadata = pd.read_csv("/kaggle/input/covid19-forecasting-metadata/region_date_metadata.csv" )
feature_day = [1,20,50,100,200,500,1000]
def CreateInput(data):
feature = []
for day in feature_day:
data.loc[:,'Number day from ' + str(day)+ ' case'] = 0
if(train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['ConfirmedCases'] < day)]['Date'].count() > 0):
fromday = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['ConfirmedCases'] < day)]['Date'].max()
else:
fromday = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]['Date'].min()
for i in range(0, len(data)) :
if(data['Date'].iloc[i] > fromday):
day_denta = data['Date'].iloc[i] - fromday
data['Number day from ' + str(day)+ ' case'].iloc[i] = day_denta.days
feature = feature + ['Number day from ' + str(day)+ ' case']
return data[feature]
pred_data_all = pd.DataFrame()
for country in train['Country_Region'].unique() :
for province in train[(train['Country_Region'] == country)]['Province_State'].unique() :
df_train = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]
df_test = test[(test['Country_Region'] == country)&(test['Province_State'] == province)]
X_train = CreateInput(df_train)
 y_train_confirmed = df_train['ConfirmedCases'].ravel()
y_train_fatalities = df_train['Fatalities'].ravel()
X_pred = CreateInput(df_test)
 for day in sorted(feature_day,reverse = True):
feature_use = 'Number day from ' + str(day)+ ' case'
idx = X_train[X_train[feature_use] == 0].shape[0]
if(X_train[X_train[feature_use] > 0].shape[0] >= 20):
break
adjusted_X_train = X_train[idx:][feature_use].values.reshape(-1, 1)
 adjusted_y_train_confirmed = y_train_confirmed[idx:]
adjusted_y_train_fatalities = y_train_fatalities[idx:]
idx = X_pred[X_pred[feature_use] == 0].shape[0]
adjusted_X_pred = X_pred[idx:][feature_use].values.reshape(-1, 1)
 pred_data = test[(test['Country_Region'] == country)&(test['Province_State'] == province)]
max_train_date = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]['Date'].max()
min_test_date = pred_data['Date'].min()
model = ExponentialSmoothing(adjusted_y_train_confirmed, trend = 'additive' ).fit()
y_hat_confirmed = model.forecast(pred_data[pred_data['Date'] > max_train_date].shape[0])
 y_train_confirmed = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['Date'] >= min_test_date)]['ConfirmedCases'].values
y_hat_confirmed = np.concatenate(( y_train_confirmed,y_hat_confirmed), axis = 0)
 model = ExponentialSmoothing(adjusted_y_train_fatalities, trend = 'additive' ).fit()
y_hat_fatalities = model.forecast(pred_data[pred_data['Date'] > max_train_date].shape[0])
 y_train_fatalities = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['Date'] >= min_test_date)]['Fatalities'].values
y_hat_fatalities = np.concatenate(( y_train_fatalities,y_hat_fatalities), axis = 0)
 pred_data['ConfirmedCases_hat'] = y_hat_confirmed
pred_data['Fatalities_hat'] = y_hat_fatalities
pred_data_all = pred_data_all.append(pred_data)
 df_val = pd.merge(pred_data_all,train[['Date','Country_Region','Province_State','ConfirmedCases','Fatalities']],on=['Date','Country_Region','Province_State'], how='left')
 df_val.loc[df_val['Fatalities_hat'] < 0,'Fatalities_hat'] = 0
df_val.loc[df_val['ConfirmedCases_hat'] < 0,'ConfirmedCases_hat'] = 0
df_val_2 = df_val.copy()<feature_engineering>
train = train.merge(test[["ForecastId", "Province_State", "Country_Region", "Date"]], on = ["Province_State", "Country_Region", "Date"], how = "left")
 display(train.head())
 test = test[~test.Date.isin(train.Date.unique())]
display(test.head())
 df = pd.concat([train, test], sort = False)
 df.head()
feature_day = [1,20,50,100,200,500,1000]
def CreateInput(data):
feature = []
for day in feature_day:
data.loc[:,'Number day from ' + str(day)+ ' case'] = 0
if(train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['ConfirmedCases'] < day)]['Date'].count() > 0):
fromday = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['ConfirmedCases'] < day)]['Date'].max()
else:
fromday = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]['Date'].min()
for i in range(0, len(data)) :
if(data['Date'].iloc[i] > fromday):
day_denta = data['Date'].iloc[i] - fromday
data['Number day from ' + str(day)+ ' case'].iloc[i] = day_denta.days
feature = feature + ['Number day from ' + str(day)+ ' case']
return data[feature]
pred_data_all = pd.DataFrame()
for country in train['Country_Region'].unique() :
for province in train[(train['Country_Region'] == country)]['Province_State'].unique() :
df_train = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]
df_test = test[(test['Country_Region'] == country)&(test['Province_State'] == province)]
X_train = CreateInput(df_train)
 y_train_confirmed = df_train['ConfirmedCases'].ravel()
y_train_fatalities = df_train['Fatalities'].ravel()
X_pred = CreateInput(df_test)
 for day in sorted(feature_day,reverse = True):
feature_use = 'Number day from ' + str(day)+ ' case'
idx = X_train[X_train[feature_use] == 0].shape[0]
if(X_train[X_train[feature_use] > 0].shape[0] >= 20):
break
adjusted_X_train = X_train[idx:][feature_use].values.reshape(-1, 1)
 adjusted_y_train_confirmed = y_train_confirmed[idx:]
adjusted_y_train_fatalities = y_train_fatalities[idx:]
idx = X_pred[X_pred[feature_use] == 0].shape[0]
adjusted_X_pred = X_pred[idx:][feature_use].values.reshape(-1, 1)
 pred_data = test[(test['Country_Region'] == country)&(test['Province_State'] == province)]
max_train_date = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]['Date'].max()
min_test_date = pred_data['Date'].min()
model = SARIMAX(adjusted_y_train_confirmed, order=(1,1,0),
measurement_error=True ).fit(disp=False)
 y_hat_confirmed = model.forecast(pred_data[pred_data['Date'] > max_train_date].shape[0])
 y_train_confirmed = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['Date'] >= min_test_date)]['ConfirmedCases'].values
y_hat_confirmed = np.concatenate(( y_train_confirmed,y_hat_confirmed), axis = 0)
 model = SARIMAX(adjusted_y_train_fatalities, order=(1,1,0),
measurement_error=True ).fit(disp=False)
 y_hat_fatalities = model.forecast(pred_data[pred_data['Date'] > max_train_date].shape[0])
 y_train_fatalities = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['Date'] >= min_test_date)]['Fatalities'].values
y_hat_fatalities = np.concatenate(( y_train_fatalities,y_hat_fatalities), axis = 0)
 pred_data['ConfirmedCases_hat'] = y_hat_confirmed
pred_data['Fatalities_hat'] = y_hat_fatalities
pred_data_all = pred_data_all.append(pred_data)
 df_val = pd.merge(pred_data_all,train[['Date','Country_Region','Province_State','ConfirmedCases','Fatalities']],on=['Date','Country_Region','Province_State'], how='left')
 df_val.loc[df_val['Fatalities_hat'] < 0,'Fatalities_hat'] = 0
df_val.loc[df_val['ConfirmedCases_hat'] < 0,'ConfirmedCases_hat'] = 0
df_val_3 = df_val.copy()<compute_test_metric>
df["geo"] = df.Country_Region.astype(str)+ ": " + df.Province_State.astype(str)
 df.loc[df.Province_State.isna() , "geo"] = df[df.Province_State.isna() ].Country_Region
df.ConfirmedCases = df.groupby("geo")["ConfirmedCases"].cummax()
df.Fatalities = df.groupby("geo")["Fatalities"].cummax()
df = df.merge(region_metadata, on = ["Country_Region", "Province_State"])
 df = df.merge(region_date_metadata, on = ["Country_Region", "Province_State", "Date"], how = "left")
 df.continent = LabelEncoder().fit_transform(df.continent)
 df.Date = pd.to_datetime(df.Date, format = "%Y-%m-%d")
 df.sort_values(["geo", "Date"], inplace = True)
 df.head()
method_list = ['Poly Bayesian Ridge','Exponential Smoothing','SARIMA']
method_val = [df_val_1,df_val_2,df_val_3]
for i in range(0,3):
df_val = method_val[i]
method_score = [method_list[i]] + [RMSLE(df_val[(df_val['ConfirmedCases'].isnull() == False)]['ConfirmedCases'].values,df_val[(df_val['ConfirmedCases'].isnull() == False)]['ConfirmedCases_hat'].values)] + [RMSLE(df_val[(df_val['Fatalities'].isnull() == False)]['Fatalities'].values,df_val[(df_val['Fatalities'].isnull() == False)]['Fatalities_hat'].values)]
print(method_score )<save_to_csv>
DAYS_SINCE_CASES = [1, 10, 50, 100, 500, 1000, 5000, 10000]
min_date_train = np.min(df[~df.Id.isna() ].Date)
 max_date_train = np.max(df[~df.Id.isna() ].Date)
 min_date_test = np.min(df[~df.ForecastId.isna() ].Date)
 max_date_test = np.max(df[~df.ForecastId.isna() ].Date)
 n_dates_test = len(df[~df.ForecastId.isna() ].Date.unique())
 print("Train date range:", str(min_date_train), " - ", str(max_date_train))
print("Test date range:", str(min_date_test), " - ", str(max_date_test))
for lag in range(1, 41):
df[f"lag_{lag}_cc"] = df.groupby("geo")["ConfirmedCases"].shift(lag)
 df[f"lag_{lag}_ft"] = df.groupby("geo")["Fatalities"].shift(lag)
 df[f"lag_{lag}_rc"] = df.groupby("geo")["Recoveries"].shift(lag)
 for case in DAYS_SINCE_CASES:
df = df.merge(df[df.ConfirmedCases >= case].groupby("geo")["Date"].min().reset_index().rename(columns = {"Date": f"case_{case}_date"}), on = "geo", how = "left" )
df_val = df_val_3
submission = df_val[['ForecastId','ConfirmedCases_hat','Fatalities_hat']]
submission.columns = ['ForecastId','ConfirmedCases','Fatalities']
submission.to_csv('submission.csv', index=False)
 submission<filter>
def prepare_features(df, gap):
df["perc_1_ac"] =(df[f"lag_{gap}_cc"] - df[f"lag_{gap}_ft"] - df[f"lag_{gap}_rc"])/ df[f"lag_{gap}_cc"]
df["perc_1_cc"] = df[f"lag_{gap}_cc"] / df.population
df["diff_1_cc"] = df[f"lag_{gap}_cc"] - df[f"lag_{gap + 1}_cc"]
df["diff_2_cc"] = df[f"lag_{gap + 1}_cc"] - df[f"lag_{gap + 2}_cc"]
df["diff_3_cc"] = df[f"lag_{gap + 2}_cc"] - df[f"lag_{gap + 3}_cc"]
df["diff_1_ft"] = df[f"lag_{gap}_ft"] - df[f"lag_{gap + 1}_ft"]
df["diff_2_ft"] = df[f"lag_{gap + 1}_ft"] - df[f"lag_{gap + 2}_ft"]
df["diff_3_ft"] = df[f"lag_{gap + 2}_ft"] - df[f"lag_{gap + 3}_ft"]
df["diff_123_cc"] =(df[f"lag_{gap}_cc"] - df[f"lag_{gap + 3}_cc"])/ 3
df["diff_123_ft"] =(df[f"lag_{gap}_ft"] - df[f"lag_{gap + 3}_ft"])/ 3
df["diff_change_1_cc"] = df.diff_1_cc / df.diff_2_cc
df["diff_change_2_cc"] = df.diff_2_cc / df.diff_3_cc
df["diff_change_1_ft"] = df.diff_1_ft / df.diff_2_ft
df["diff_change_2_ft"] = df.diff_2_ft / df.diff_3_ft
df["diff_change_12_cc"] =(df.diff_change_1_cc + df.diff_change_2_cc)/ 2
df["diff_change_12_ft"] =(df.diff_change_1_ft + df.diff_change_2_ft)/ 2
df["change_1_cc"] = df[f"lag_{gap}_cc"] / df[f"lag_{gap + 1}_cc"]
df["change_2_cc"] = df[f"lag_{gap + 1}_cc"] / df[f"lag_{gap + 2}_cc"]
df["change_3_cc"] = df[f"lag_{gap + 2}_cc"] / df[f"lag_{gap + 3}_cc"]
df["change_1_ft"] = df[f"lag_{gap}_ft"] / df[f"lag_{gap + 1}_ft"]
df["change_2_ft"] = df[f"lag_{gap + 1}_ft"] / df[f"lag_{gap + 2}_ft"]
df["change_3_ft"] = df[f"lag_{gap + 2}_ft"] / df[f"lag_{gap + 3}_ft"]
df["change_123_cc"] = df[f"lag_{gap}_cc"] / df[f"lag_{gap + 3}_cc"]
df["change_123_ft"] = df[f"lag_{gap}_ft"] / df[f"lag_{gap + 3}_ft"]
for case in DAYS_SINCE_CASES:
df[f"days_since_{case}_case"] =(df[f"case_{case}_date"] - df.Date ).astype("timedelta64[D]")
 df.loc[df[f"days_since_{case}_case"] < gap, f"days_since_{case}_case"] = np.nan
df["country_flag"] = df.Province_State.isna().astype(int)
 df["density"] = df.population / df.area
df["target_cc"] = np.log1p(df.ConfirmedCases)- np.log1p(df[f"lag_{gap}_cc"])
 df["target_ft"] = np.log1p(df.Fatalities)- np.log1p(df[f"lag_{gap}_ft"])
 features = [
f"lag_{gap}_cc",
f"lag_{gap}_ft",
f"lag_{gap}_rc",
"perc_1_ac",
"perc_1_cc",
"diff_1_cc",
"diff_2_cc",
"diff_3_cc",
"diff_1_ft",
"diff_2_ft",
"diff_3_ft",
"diff_123_cc",
"diff_123_ft",
"diff_change_1_cc",
"diff_change_2_cc",
"diff_change_1_ft",
"diff_change_2_ft",
"diff_change_12_cc",
"diff_change_12_ft",
"change_1_cc",
"change_2_cc",
"change_3_cc",
"change_1_ft",
"change_2_ft",
"change_3_ft",
"change_123_cc",
"change_123_ft",
"days_since_1_case",
"days_since_10_case",
"days_since_50_case",
"days_since_100_case",
"days_since_500_case",
"days_since_1000_case",
"days_since_5000_case",
"days_since_10000_case",
"country_flag",
"lat",
"lon",
"continent",
"population",
"area",
"density",
"target_cc",
"target_ft"
]
return df[features]
df_worldinfor[df_worldinfor['Country'] == 'Vietnam']<compute_test_metric>
def build_predict_lgbm(df_train, df_test, gap):
df_train.dropna(subset = ["target_cc", "target_ft", f"lag_{gap}_cc", f"lag_{gap}_ft"], inplace = True)
 target_cc = df_train.target_cc
target_ft = df_train.target_ft
test_lag_cc = df_test[f"lag_{gap}_cc"].values
test_lag_ft = df_test[f"lag_{gap}_ft"].values
df_train.drop(["target_cc", "target_ft"], axis = 1, inplace = True)
 df_test.drop(["target_cc", "target_ft"], axis = 1, inplace = True)
 categorical_features = ["continent"]
dtrain_cc = lgb.Dataset(df_train, label = target_cc, categorical_feature = categorical_features)
 dtrain_ft = lgb.Dataset(df_train, label = target_ft, categorical_feature = categorical_features)
 model_cc = lgb.train(LGB_PARAMS, train_set = dtrain_cc, num_boost_round = 200)
 model_ft = lgb.train(LGB_PARAMS, train_set = dtrain_ft, num_boost_round = 200)
 y_pred_cc = np.expm1(model_cc.predict(df_test, num_boost_round = 200)+ np.log1p(test_lag_cc))
y_pred_ft = np.expm1(model_ft.predict(df_test, num_boost_round = 200)+ np.log1p(test_lag_ft))
return y_pred_cc, y_pred_ft, model_cc, model_ft
def RMSLE(pred,actual):
return np.sqrt(np.mean(np.power(( np.log(pred+1)-np.log(actual+1)) ,2)) )<load_from_csv>
def predict_mad(df_test, gap, val = False):
df_test["avg_diff_cc"] =(df_test[f"lag_{gap}_cc"] - df_test[f"lag_{gap + 3}_cc"])/ 3
df_test["avg_diff_ft"] =(df_test[f"lag_{gap}_ft"] - df_test[f"lag_{gap + 3}_ft"])/ 3
if val:
y_pred_cc = df_test[f"lag_{gap}_cc"] + gap * df_test.avg_diff_cc -(1 - MAD_FACTOR)* df_test.avg_diff_cc * np.sum([x for x in range(gap)])/ VAL_DAYS
y_pred_ft = df_test[f"lag_{gap}_ft"] + gap * df_test.avg_diff_ft -(1 - MAD_FACTOR)* df_test.avg_diff_ft * np.sum([x for x in range(gap)])/ VAL_DAYS
else:
y_pred_cc = df_test[f"lag_{gap}_cc"] + gap * df_test.avg_diff_cc -(1 - MAD_FACTOR)* df_test.avg_diff_cc * np.sum([x for x in range(gap)])/ n_dates_test
y_pred_ft = df_test[f"lag_{gap}_ft"] + gap * df_test.avg_diff_ft -(1 - MAD_FACTOR)* df_test.avg_diff_ft * np.sum([x for x in range(gap)])/ n_dates_test
return y_pred_cc, y_pred_ft
pd.set_option('mode.chained_assignment', None)
 test = pd.read_csv(".. /input/covid19-global-forecasting-week-2/test.csv")
 train = pd.read_csv(".. /input/covid19-global-forecasting-week-2/train.csv")
 train['Province_State'].fillna('', inplace=True)
 test['Province_State'].fillna('', inplace=True)
 train['Date'] = pd.to_datetime(train['Date'])
 test['Date'] = pd.to_datetime(test['Date'])
 train = train.sort_values(['Country_Region','Province_State','Date'])
 test = test.sort_values(['Country_Region','Province_State','Date'] )<feature_engineering>
SEED = 23
LGB_PARAMS = {"objective": "regression",
"num_leaves": 5,
"learning_rate": 0.013,
"bagging_fraction": 0.91,
"feature_fraction": 0.81,
"reg_alpha": 0.13,
"reg_lambda": 0.13,
"metric": "rmse",
"seed": SEED
}
VAL_DAYS = 7
MAD_FACTOR = 0.5
feature_day = [1,20,50,100,200,500,1000]
def CreateInput(data):
feature = []
for day in feature_day:
data.loc[:,'Number day from ' + str(day)+ ' case'] = 0
if(train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['ConfirmedCases'] < day)]['Date'].count() > 0):
fromday = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['ConfirmedCases'] < day)]['Date'].max()
else:
fromday = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]['Date'].min()
for i in range(0, len(data)) :
if(data['Date'].iloc[i] > fromday):
day_denta = data['Date'].iloc[i] - fromday
data['Number day from ' + str(day)+ ' case'].iloc[i] = day_denta.days
feature = feature + ['Number day from ' + str(day)+ ' case']
return data[feature]
pred_data_all = pd.DataFrame()
for country in train['Country_Region'].unique() :
for province in train[(train['Country_Region'] == country)]['Province_State'].unique() :
df_train = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]
df_test = test[(test['Country_Region'] == country)&(test['Province_State'] == province)]
X_train = CreateInput(df_train)
 y_train_confirmed = df_train['ConfirmedCases'].ravel()
y_train_fatalities = df_train['Fatalities'].ravel()
X_pred = CreateInput(df_test)
 for day in sorted(feature_day,reverse = True):
feature_use = 'Number day from ' + str(day)+ ' case'
idx = X_train[X_train[feature_use] == 0].shape[0]
if(X_train[X_train[feature_use] > 0].shape[0] >= 10):
break
adjusted_X_train = X_train[idx:][feature_use].values.reshape(-1, 1)
 adjusted_y_train_confirmed = y_train_confirmed[idx:]
adjusted_y_train_fatalities = y_train_fatalities[idx:]
idx = X_pred[X_pred[feature_use] == 0].shape[0]
adjusted_X_pred = X_pred[idx:][feature_use].values.reshape(-1, 1)
 model = make_pipeline(PolynomialFeatures(2), BayesianRidge())
 model.fit(adjusted_X_train,adjusted_y_train_confirmed)
 y_hat_confirmed = model.predict(adjusted_X_pred)
 model.fit(adjusted_X_train,adjusted_y_train_fatalities)
 y_hat_fatalities = model.predict(adjusted_X_pred)
 pred_data = test[(test['Country_Region'] == country)&(test['Province_State'] == province)]
pred_data['ConfirmedCases_hat'] = np.concatenate(( np.repeat(0, len(pred_data)- len(y_hat_confirmed)) , y_hat_confirmed), axis = 0)
 pred_data['Fatalities_hat'] = np.concatenate(( np.repeat(float(0), len(pred_data)- len(y_hat_fatalities)) , y_hat_fatalities), axis = 0)
 pred_data_all = pred_data_all.append(pred_data)
 df_val = pd.merge(pred_data_all,train[['Date','Country_Region','Province_State','ConfirmedCases','Fatalities']],on=['Date','Country_Region','Province_State'], how='left' )<feature_engineering>
df_train = df[~df.Id.isna() ]
df_test_full = df[~df.ForecastId.isna() ]
df_val.loc[df_val['Fatalities_hat'] < 0,'Fatalities_hat'] = 0
df_val.loc[df_val['ConfirmedCases_hat'] < 0,'ConfirmedCases_hat'] = 0<compute_test_metric>
df_preds_val = []
df_preds_test = []
for date in df_test_full.Date.unique() :
print("[INFO] Date:", date)
 if date in df_train.Date.values:
df_pred_test = df_test_full.loc[df_test_full.Date == date, ["ForecastId", "ConfirmedCases", "Fatalities"]].rename(columns = {"ConfirmedCases": "ConfirmedCases_test", "Fatalities": "Fatalities_test"})
 else:
df_test = df_test_full[df_test_full.Date == date]
gap =(pd.Timestamp(date)- max_date_train ).days
if gap <= VAL_DAYS:
val_date = max_date_train - pd.Timedelta(VAL_DAYS, "D")+ pd.Timedelta(gap, "D")
 df_build = df_train[df_train.Date < val_date]
df_val = df_train[df_train.Date == val_date]
X_build = prepare_features(df_build, gap)
 X_val = prepare_features(df_val, gap)
 y_val_cc_lgb, y_val_ft_lgb, _, _ = build_predict_lgbm(X_build, X_val, gap)
 y_val_cc_mad, y_val_ft_mad = predict_mad(df_val, gap, val = True)
 df_pred_val = pd.DataFrame({"Id": df_val.Id.values,
"ConfirmedCases_val_lgb": y_val_cc_lgb,
"Fatalities_val_lgb": y_val_ft_lgb,
"ConfirmedCases_val_mad": y_val_cc_mad,
"Fatalities_val_mad": y_val_ft_mad,
})
 df_preds_val.append(df_pred_val)
 X_train = prepare_features(df_train, gap)
 X_test = prepare_features(df_test, gap)
 y_test_cc_lgb, y_test_ft_lgb, model_cc, model_ft = build_predict_lgbm(X_train, X_test, gap)
 y_test_cc_mad, y_test_ft_mad = predict_mad(df_test, gap)
 if gap == 1:
model_1_cc = model_cc
model_1_ft = model_ft
features_1 = X_train.columns.values
elif gap == 14:
model_14_cc = model_cc
model_14_ft = model_ft
features_14 = X_train.columns.values
elif gap == 28:
model_28_cc = model_cc
model_28_ft = model_ft
features_28 = X_train.columns.values
df_pred_test = pd.DataFrame({"ForecastId": df_test.ForecastId.values,
"ConfirmedCases_test_lgb": y_test_cc_lgb,
"Fatalities_test_lgb": y_test_ft_lgb,
"ConfirmedCases_test_mad": y_test_cc_mad,
"Fatalities_test_mad": y_test_ft_mad,
})
 df_preds_test.append(df_pred_test )
RMSLE(df_val[(df_val['ConfirmedCases'].isnull() == False)]['ConfirmedCases'].values,df_val[(df_val['ConfirmedCases'].isnull() == False)]['ConfirmedCases_hat'].values )<compute_test_metric>
df = df.merge(pd.concat(df_preds_val, sort = False), on = "Id", how = "left")
 df = df.merge(pd.concat(df_preds_test, sort = False), on = "ForecastId", how = "left")
 rmsle_cc_lgb = np.sqrt(mean_squared_error(np.log1p(df[~df.ConfirmedCases_val_lgb.isna() ].ConfirmedCases), np.log1p(df[~df.ConfirmedCases_val_lgb.isna() ].ConfirmedCases_val_lgb)))
 rmsle_ft_lgb = np.sqrt(mean_squared_error(np.log1p(df[~df.Fatalities_val_lgb.isna() ].Fatalities), np.log1p(df[~df.Fatalities_val_lgb.isna() ].Fatalities_val_lgb)))
 rmsle_cc_mad = np.sqrt(mean_squared_error(np.log1p(df[~df.ConfirmedCases_val_mad.isna() ].ConfirmedCases), np.log1p(df[~df.ConfirmedCases_val_mad.isna() ].ConfirmedCases_val_mad)))
 rmsle_ft_mad = np.sqrt(mean_squared_error(np.log1p(df[~df.Fatalities_val_mad.isna() ].Fatalities), np.log1p(df[~df.Fatalities_val_mad.isna() ].Fatalities_val_mad)))
 print("LGB CC RMSLE Val of", VAL_DAYS, "days for CC:", round(rmsle_cc_lgb, 2))
print("LGB FT RMSLE Val of", VAL_DAYS, "days for FT:", round(rmsle_ft_lgb, 2))
print("LGB Overall RMSLE Val of", VAL_DAYS, "days:", round(( rmsle_cc_lgb + rmsle_ft_lgb)/ 2, 2))
print("MAD CC RMSLE Val of", VAL_DAYS, "days for CC:", round(rmsle_cc_mad, 2))
print("MAD FT RMSLE Val of", VAL_DAYS, "days for FT:", round(rmsle_ft_mad, 2))
print("MAD Overall RMSLE Val of", VAL_DAYS, "days:", round(( rmsle_cc_mad + rmsle_ft_mad)/ 2, 2))
RMSLE(df_val[(df_val['Fatalities'].isnull() == False)]['Fatalities'].values,df_val[(df_val['Fatalities'].isnull() == False)]['Fatalities_hat'].values )<save_to_csv>
test = df.loc[~df.ForecastId.isna() , ["ForecastId", "Country_Region", "Province_State", "Date",
"ConfirmedCases_test", "ConfirmedCases_test_lgb", "ConfirmedCases_test_mad",
"Fatalities_test", "Fatalities_test_lgb", "Fatalities_test_mad"]].reset_index()
test["ConfirmedCases"] = 0.3 * test.ConfirmedCases_test_lgb + 0.7 * test.ConfirmedCases_test_mad
test["Fatalities"] = 0.25 * test.Fatalities_test_lgb + 0.75 * test.Fatalities_test_mad
test.loc[test.Country_Region.isin(["China", "US", "Diamond Princess"]), "ConfirmedCases"] = test[test.Country_Region.isin(["China", "US", "Diamond Princess"])].ConfirmedCases_test_mad.values
test.loc[test.Country_Region.isin(["China", "US", "Diamond Princess"]), "Fatalities"] = test[test.Country_Region.isin(["China", "US", "Diamond Princess"])].Fatalities_test_mad.values
test.loc[test.Date.isin(df_train.Date.values), "ConfirmedCases"] = test[test.Date.isin(df_train.Date.values)].ConfirmedCases_test.values
test.loc[test.Date.isin(df_train.Date.values), "Fatalities"] = test[test.Date.isin(df_train.Date.values)].Fatalities_test.values
sub0 = test[["ForecastId", "ConfirmedCases", "Fatalities"]]
sub0.ForecastId = sub0.ForecastId.astype(int)
 sub0.head()
submission = df_val[['ForecastId','ConfirmedCases_hat','Fatalities_hat']]
submission.columns = ['ForecastId','ConfirmedCases','Fatalities']
submission.to_csv('submission.csv', index=False )<feature_engineering>
test = pd.read_csv(".. /input/covid19-global-forecasting-week-3/test.csv")
 train = pd.read_csv(".. /input/covid19-global-forecasting-week-3/train.csv")
 train['Province_State'].fillna('', inplace=True)
 test['Province_State'].fillna('', inplace=True)
 train['Date'] = pd.to_datetime(train['Date'])
 test['Date'] = pd.to_datetime(test['Date'])
 train = train.sort_values(['Country_Region','Province_State','Date'])
 test = test.sort_values(['Country_Region','Province_State','Date'] )
feature_day = [1,20,50,100,200,500,1000]
def CreateInput(data):
feature = []
for day in feature_day:
data.loc[:,'Number day from ' + str(day)+ ' case'] = 0
if(train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['ConfirmedCases'] < day)]['Date'].count() > 0):
fromday = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['ConfirmedCases'] < day)]['Date'].max()
else:
fromday = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]['Date'].min()
for i in range(0, len(data)) :
if(data['Date'].iloc[i] > fromday):
day_denta = data['Date'].iloc[i] - fromday
data['Number day from ' + str(day)+ ' case'].iloc[i] = day_denta.days
feature = feature + ['Number day from ' + str(day)+ ' case']
return data[feature]
pred_data_all = pd.DataFrame()
for country in train['Country_Region'].unique() :
for province in train[(train['Country_Region'] == country)]['Province_State'].unique() :
df_train = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]
df_test = test[(test['Country_Region'] == country)&(test['Province_State'] == province)]
X_train = CreateInput(df_train)
 y_train_confirmed = df_train['ConfirmedCases'].ravel()
y_train_fatalities = df_train['Fatalities'].ravel()
X_pred = CreateInput(df_test)
 for day in sorted(feature_day,reverse = True):
feature_use = 'Number day from ' + str(day)+ ' case'
idx = X_train[X_train[feature_use] == 0].shape[0]
if(X_train[X_train[feature_use] > 0].shape[0] >= 20):
break
adjusted_X_train = X_train[idx:][feature_use].values.reshape(-1, 1)
 adjusted_y_train_confirmed = y_train_confirmed[idx:]
adjusted_y_train_fatalities = y_train_fatalities[idx:]
idx = X_pred[X_pred[feature_use] == 0].shape[0]
adjusted_X_pred = X_pred[idx:][feature_use].values.reshape(-1, 1)
 pred_data = test[(test['Country_Region'] == country)&(test['Province_State'] == province)]
max_train_date = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]['Date'].max()
min_test_date = pred_data['Date'].min()
model = ExponentialSmoothing(adjusted_y_train_confirmed, trend = 'additive' ).fit()
y_hat_confirmed = model.forecast(pred_data[pred_data['Date'] > max_train_date].shape[0])
 y_train_confirmed = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['Date'] >= min_test_date)]['ConfirmedCases'].values
y_hat_confirmed = np.concatenate(( y_train_confirmed,y_hat_confirmed), axis = 0)
 model = ExponentialSmoothing(adjusted_y_train_fatalities, trend = 'additive' ).fit()
y_hat_fatalities = model.forecast(pred_data[pred_data['Date'] > max_train_date].shape[0])
 y_train_fatalities = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['Date'] >= min_test_date)]['Fatalities'].values
y_hat_fatalities = np.concatenate(( y_train_fatalities,y_hat_fatalities), axis = 0)
 pred_data['ConfirmedCases_hat'] = y_hat_confirmed
pred_data['Fatalities_hat'] = y_hat_fatalities
pred_data_all = pred_data_all.append(pred_data)
 df_val = pd.merge(pred_data_all,train[['Date','Country_Region','Province_State','ConfirmedCases','Fatalities']],on=['Date','Country_Region','Province_State'], how='left')
 df_val 
<import_modules>
FirstDate = train.groupby('Country_Region' ).min() ['Date'].unique() [0]
train['Last Confirm'] = train['ConfirmedCases'].shift(1)
 while train[(train['Last Confirm'] > train['ConfirmedCases'])&(train['Date'] > FirstDate)].shape[0] > 0:
train['Last Confirm'] = train['ConfirmedCases'].shift(1)
 train['Last Fatalities'] = train['Fatalities'].shift(1)
 train.loc[(train['Last Confirm'] > train['ConfirmedCases'])&(train['Date'] > FirstDate),'ConfirmedCases'] = train.loc[(train['Last Confirm'] > train['ConfirmedCases'])&(train['Date'] > FirstDate),'Last Confirm']
train.loc[(train['Last Fatalities'] > train['Fatalities'])&(train['Date'] > FirstDate),'Fatalities'] = train.loc[(train['Last Fatalities'] > train['Fatalities'])&(train['Date'] > FirstDate),'Last Fatalities']
train['Last Confirm'] = train['ConfirmedCases'].shift(1)
 train['Last Fatalities'] = train['Fatalities'].shift(1 )
def create_time_features(df):

 df['date'] = pd.to_datetime(df['Date'] ).values
df['hour'] = df['date'].dt.hour
df['dayofweek'] = df['date'].dt.dayofweek
df['quarter'] = df['date'].dt.quarter
df['month'] = df['date'].dt.month
df['year'] = df['date'].dt.year
df['dayofyear'] = df['date'].dt.dayofyear
df['dayofmonth'] = df['date'].dt.day
df['weekofyear'] = df['date'].dt.weekofyear
return df
def mean_absolute_percentage_error(y_true, y_pred):
y_true, y_pred = np.array(y_true), np.array(y_pred)
 return np.mean(np.abs(( y_true - y_pred)/ y_true)) * 10
def wg_func(params):
a = params[0]
r = params[1]
c = params[2]
hcnb = params[3]
incr = [cdata[i] if i == 0 else cdata[i] - cdata[i-1] for i,item in enumerate(cdata)]
ptlt = [(1-(a/(a+item**c)) **r)*(1-hcnb)for item in ts]
ptwt = [ptlt[i] if i == 0 else ptlt[i] - ptlt[i-1] for i,item in enumerate(ptlt)]
return(np.sum([j * np.log(i)for i,j in zip(ptwt,incr)])
 +(pop*(1-hcnb)- np.max(cdata)) * np.log(1-np.max(ptlt))
+ np.log(hcnb)*pop*hcnb)*-1
def wg_func2(params):
a = params[0]
r = params[1]
c = params[2]
hcnb =.98
incr = [cdata[i] if i == 0 else cdata[i] - cdata[i-1] for i,item in enumerate(cdata)]
ptlt = [(1-(a/(a+item**c)) **r)*(1-hcnb)for item in ts]
ptwt = [ptlt[i] if i == 0 else ptlt[i] - ptlt[i-1] for i,item in enumerate(ptlt)]
return(np.sum([j * np.log(i)for i,j in zip(ptwt,incr)])
 +(pop*(1-hcnb)- np.max(cdata)) * np.log(1-np.max(ptlt))
+ np.log(hcnb)*pop*hcnb)*-1
def constraint1(inputs):
return inputs[0]
cons =({'type': 'ineq', "fun": constraint1})
 def calc_distance(lat1, lng1, lat2, lng2):
R = 6373.0
lat1 = radians(lat1)
 lng1 = radians(lng1)
 lat2 = radians(lat2)
 lng2 = radians(lng2)
 dlon = lng2 - lng1
dlat = lat2 - lat1
a = sin(dlat / 2)**2 + cos(lat1)* cos(lat2)* sin(dlon / 2)**2
c = 2 * atan2(sqrt(a), sqrt(1 - a))
return R * c
def fill_missing_coords(df):
print(' Filling in missing lat,lng')
 df.loc[df.Country_Region=='Zimbabwe', 'Lat'] = 19.0154
df.loc[df.Country_Region=='Zimbabwe', 'Long']= 29.1549
df.loc[(df.Country_Region=='Angola')&(df.Province_State==''), 'Lat'] = -11.2027
df.loc[(df.Country_Region=='Angola')&(df.Province_State==''), 'Long']= 17.8739
df.loc[(df.Country_Region=='Bahamas')&(df.Province_State==''), 'Lat'] = 25.0343
df.loc[(df.Country_Region=='Bahamas')&(df.Province_State==''), 'Long']= -77.3963
df.loc[(df.Country_Region=='Belize')&(df.Province_State==''), 'Lat'] = 17.1899
df.loc[(df.Country_Region=='Belize')&(df.Province_State==''), 'Long']= -88.4976
df.loc[(df.Country_Region=='United Kingdom')&(df.Province_State==''), 'Lat'] = 55.3781
df.loc[(df.Country_Region=='United Kingdom')&(df.Province_State==''), 'Long']= -3.4360
df.loc[(df.Country_Region=='United Kingdom')&(df.Province_State=='Isle of Man'), 'Lat'] = 54.2361
df.loc[(df.Country_Region=='United Kingdom')&(df.Province_State=='Isle of Man'), 'Long']= -4.5481
df.loc[(df.Country_Region=='Cabo Verde')&(df.Province_State==''), 'Lat'] = 16.5388
df.loc[(df.Country_Region=='Cabo Verde')&(df.Province_State==''), 'Long']= -23.0418
df.loc[(df.Country_Region=='United Kingdom')&(df.Province_State=='Bermuda'), 'Lat'] = 32.3078
df.loc[(df.Country_Region=='United Kingdom')&(df.Province_State=='Bermuda'), 'Long']= -64.7505
df.loc[(df.Country_Region=='Chad')&(df.Province_State==''), 'Lat'] = 15.4542
df.loc[(df.Country_Region=='Chad')&(df.Province_State==''), 'Long']= 18.7322
df.loc[(df.Country_Region=='Uganda')&(df.Province_State==''), 'Lat'] = 1.3733
df.loc[(df.Country_Region=='Uganda')&(df.Province_State==''), 'Long']= 32.2903
df.loc[(df.Country_Region=='Denmark')&(df.Province_State=='Greenland'), 'Lat'] = 71.7069
df.loc[(df.Country_Region=='Denmark')&(df.Province_State=='Greenland'), 'Long']= -42.6043
df.loc[(df.Country_Region=='Denmark')&(df.Province_State==''), 'Lat'] = 56.2639
df.loc[(df.Country_Region=='Denmark')&(df.Province_State==''), 'Long']= 9.5018
df.loc[(df.Country_Region=='Timor-Leste')&(df.Province_State==''), 'Lat'] = -8.8742
df.loc[(df.Country_Region=='Timor-Leste')&(df.Province_State==''), 'Long']= 125.7275
df.loc[(df.Country_Region=='Syria')&(df.Province_State==''), 'Lat'] = 34.8021
df.loc[(df.Country_Region=='Syria')&(df.Province_State==''), 'Long']= 38.9968
df.loc[(df.Country_Region=='Saint Kitts and Nevis')&(df.Province_State==''), 'Lat'] = 17.3578
df.loc[(df.Country_Region=='Saint Kitts and Nevis')&(df.Province_State==''), 'Long']= -62.7830
df.loc[(df.Country_Region=='Papua New Guinea')&(df.Province_State==''), 'Lat'] = -6.3150
df.loc[(df.Country_Region=='Papua New Guinea')&(df.Province_State==''), 'Long']= 143.9555
df.loc[(df.Country_Region=='Niger')&(df.Province_State==''), 'Lat'] = 17.6078
df.loc[(df.Country_Region=='Niger')&(df.Province_State==''), 'Long']= 8.0817
df.loc[(df.Country_Region=='El Salvador')&(df.Province_State==''), 'Lat'] = 13.7942
df.loc[(df.Country_Region=='El Salvador')&(df.Province_State==''), 'Long']= -88.8965
df.loc[(df.Country_Region=='Gambia')&(df.Province_State==''), 'Lat'] = 13.4432
df.loc[(df.Country_Region=='Gambia')&(df.Province_State==''), 'Long']= -15.3101
df.loc[(df.Country_Region=='Libya')&(df.Province_State==''), 'Lat'] = 26.3351
df.loc[(df.Country_Region=='Libya')&(df.Province_State==''), 'Long']= 17.2283
df.loc[(df.Country_Region=='Mali')&(df.Province_State==''), 'Lat'] = 17.5707
df.loc[(df.Country_Region=='Mali')&(df.Province_State==''), 'Long']= -3.9962
df.loc[(df.Country_Region=='Grenada')&(df.Province_State==''), 'Lat'] = 12.1165
df.loc[(df.Country_Region=='Grenada')&(df.Province_State==''), 'Long']= -61.6790
df.loc[(df.Country_Region=='Laos')&(df.Province_State==''), 'Lat'] = 19.8563
df.loc[(df.Country_Region=='Laos')&(df.Province_State==''), 'Long']= 102.4955
df.loc[(df.Country_Region=='Madagascar')&(df.Province_State==''), 'Lat'] = -18.7669
df.loc[(df.Country_Region=='Madagascar')&(df.Province_State==''), 'Long']= 46.8691
df.loc[(df.Country_Region=='Guinea-Bissau')&(df.Province_State==''), 'Lat'] = 11.8037
df.loc[(df.Country_Region=='Guinea-Bissau')&(df.Province_State==''), 'Long']= -15.1804
df.loc[(df.Country_Region=='Fiji')&(df.Province_State==''), 'Lat'] = -17.7134
df.loc[(df.Country_Region=='Fiji')&(df.Province_State==''), 'Long']= 178.0650
df.loc[(df.Country_Region=='Nicaragua')&(df.Province_State==''), 'Lat'] = 12.8654
df.loc[(df.Country_Region=='Nicaragua')&(df.Province_State==''), 'Long']= -85.2072
df.loc[(df.Country_Region=='Eritrea')&(df.Province_State==''), 'Lat'] = 15.1794
df.loc[(df.Country_Region=='Eritrea')&(df.Province_State==''), 'Long']= 39.7823
df.loc[(df.Country_Region=='Haiti')&(df.Province_State==''), 'Lat'] = 18.9712
df.loc[(df.Country_Region=='Haiti')&(df.Province_State==''), 'Long']= -72.2852
df.loc[(df.Country_Region=='Dominica')&(df.Province_State==''), 'Lat'] = 15.4150
df.loc[(df.Country_Region=='Dominica')&(df.Province_State==''), 'Long']= -61.3710
df.loc[(df.Country_Region=='Mozambique')&(df.Province_State==''), 'Lat'] = -18.6657
df.loc[(df.Country_Region=='Mozambique')&(df.Province_State==''), 'Long']= 35.5296
df.loc[(df.Country_Region=='Netherlands')&(df.Province_State==''), 'Lat'] = 52.1326
df.loc[(df.Country_Region=='Netherlands')&(df.Province_State==''), 'Long']= 5.2913
df.loc[(df.Country_Region=='Netherlands')&(df.Province_State=='Sint Maarten'), 'Lat'] = 18.0425
df.loc[(df.Country_Region=='Netherlands')&(df.Province_State=='Sint Maarten'), 'Long']= -63.0548
df.loc[(df.Country_Region=='France')&(df.Province_State==''), 'Lat'] = 46.2276
df.loc[(df.Country_Region=='France')&(df.Province_State==''), 'Long']= 2.2137
df.loc[(df.Country_Region=='France')&(df.Province_State=='New Caledonia'), 'Lat'] = -20.9043
df.loc[(df.Country_Region=='France')&(df.Province_State=='New Caledonia'), 'Long']= 165.6180
df.loc[(df.Country_Region=='France')&(df.Province_State=='Martinique'), 'Lat'] = 14.6415
df.loc[(df.Country_Region=='France')&(df.Province_State=='Martinique'), 'Long']= -61.0242
print('done', datetime.now())
<load_from_csv>
from statsmodels.tsa.statespace.sarimax import SARIMAX
from statsmodels.tsa.arima_model import ARIMA
PATH = '/kaggle/input/covid19-global-forecasting-week-2/'
train = pd.read_csv(PATH + 'train.csv')
 test = pd.read_csv(PATH + 'test.csv')
 df_geo = pd.read_csv('.. /input/df-geo/df_geo2.csv')
 df_geo.Province_State.fillna('', inplace=True)
 dfp = pd.read_csv('/kaggle/input/population3/' + 'population.csv')
 dfp.columns = ['Province_State', 'Country_Region', 'pop']
print(train.shape, dfp.shape)
 print('Cleaning Data')
 train.loc[train['Province_State'].isnull() , 'Province_State'] = ''
test.loc[test['Province_State'].isnull() , 'Province_State'] = ''
dfp.loc[dfp['Province_State'].isnull() , 'Province_State'] = ''
print('Joining Data')
 print(train.shape, test.shape)
 n = train.shape[0]
train = pd.merge(train, dfp, on=['Country_Region','Province_State'], how='left')
 assert train.shape[0] == n
n = test.shape[0]
test = pd.merge(test, dfp, on=['Country_Region','Province_State'], how='left')
 assert test.shape[0] == n
test.reset_index(drop=True, inplace=True)
 train.reset_index(drop=True, inplace=True)
 n = train.shape[0]
train = pd.merge(train, df_geo, on=['Country_Region','Province_State'], how='left')
 assert train.shape[0] == n
n = test.shape[0]
test = pd.merge(test, df_geo, on=['Country_Region','Province_State'], how='left')
 assert test.shape[0] == n
test.reset_index(drop=True, inplace=True)
 train.reset_index(drop=True, inplace=True)
 train.loc[train['pop'].isnull() ,'pop'] = 0
test.loc[test['pop'].isnull() ,'pop'] = 0
print('
Number of countries with missing Lat/Lng: ',
train[train.Lat.isnull() ]['Country_Region'].value_counts().shape[0])
 fill_missing_coords(train)
 fill_missing_coords(test)
 print('
Number of countries with missing Lat/Lng after fixing: ',
train[train.Lat.isnull() ]['Country_Region'].value_counts().shape[0])
 print(train.shape, test.shape)
 print('
Enriching Data')
 mo = train['Date'].apply(lambda x: x[5:7])
 da = train['Date'].apply(lambda x: x[8:10])
 train['day_from_jan_first'] =(da.apply(int)
 + 31*(mo=='02')
 + 60*(mo=='03')
 + 91*(mo=='04')
)
 mo = test['Date'].apply(lambda x: x[5:7])
 da = test['Date'].apply(lambda x: x[8:10])
 test['day_from_jan_first'] =(da.apply(int)
 + 31*(mo=='02')
 + 60*(mo=='03')
 + 91*(mo=='04')
)
 create_time_features(train)
 create_time_features(test)
 print('done', datetime.now() )<categorify>
feature_day = [1,20,50,100,200,500,1000]
def CreateInput(data):
feature = []
for day in feature_day:
data.loc[:,'Number day from ' + str(day)+ ' case'] = 0
if(train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['ConfirmedCases'] < day)]['Date'].count() > 0):
fromday = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['ConfirmedCases'] < day)]['Date'].max()
else:
fromday = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]['Date'].min()
for i in range(0, len(data)) :
if(data['Date'].iloc[i] > fromday):
day_denta = data['Date'].iloc[i] - fromday
data['Number day from ' + str(day)+ ' case'].iloc[i] = day_denta.days
feature = feature + ['Number day from ' + str(day)+ ' case']
return data[feature]
pred_data_all = pd.DataFrame()
with tqdm(total=len(train['Country_Region'].unique())) as pbar:
for country in train['Country_Region'].unique() :
for province in train[(train['Country_Region'] == country)]['Province_State'].unique() :
with warnings.catch_warnings() :
warnings.filterwarnings("ignore")
 df_train = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]
df_test = test[(test['Country_Region'] == country)&(test['Province_State'] == province)]
X_train = CreateInput(df_train)
 y_train_confirmed = df_train['ConfirmedCases'].ravel()
y_train_fatalities = df_train['Fatalities'].ravel()
X_pred = CreateInput(df_test)
 feature_use = X_pred.columns[0]
for i in range(X_pred.shape[1] - 1,0,-1):
if(X_pred.iloc[0,i] > 0):
feature_use = X_pred.columns[i]
break
idx = X_train[X_train[feature_use] == 0].shape[0]
adjusted_X_train = X_train[idx:][feature_use].values.reshape(-1, 1)
 adjusted_y_train_confirmed = y_train_confirmed[idx:]
adjusted_y_train_fatalities = y_train_fatalities[idx:]
idx = X_pred[X_pred[feature_use] == 0].shape[0]
adjusted_X_pred = X_pred[idx:][feature_use].values.reshape(-1, 1)
 pred_data = test[(test['Country_Region'] == country)&(test['Province_State'] == province)]
max_train_date = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]['Date'].max()
min_test_date = pred_data['Date'].min()
model = SARIMAX(adjusted_y_train_confirmed, order=(1,1,0),
measurement_error=True ).fit(disp=False)
 y_hat_confirmed = model.forecast(pred_data[pred_data['Date'] > max_train_date].shape[0])
 y_train_confirmed = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['Date'] >= min_test_date)]['ConfirmedCases'].values
y_hat_confirmed = np.concatenate(( y_train_confirmed,y_hat_confirmed), axis = 0)
 model = SARIMAX(adjusted_y_train_fatalities, order=(1,1,0),
measurement_error=True ).fit(disp=False)
 y_hat_fatalities = model.forecast(pred_data[pred_data['Date'] > max_train_date].shape[0])
 y_train_fatalities = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['Date'] >= min_test_date)]['Fatalities'].values
y_hat_fatalities = np.concatenate(( y_train_fatalities,y_hat_fatalities), axis = 0)
 pred_data['ConfirmedCases_hat'] = y_hat_confirmed
pred_data['Fatalities_hat'] = y_hat_fatalities
pred_data_all = pred_data_all.append(pred_data)
 pbar.update(1)
 df_val = pd.merge(pred_data_all,train[['Date','Country_Region','Province_State','ConfirmedCases','Fatalities']],on=['Date','Country_Region','Province_State'], how='left')
 df_val.loc[df_val['Fatalities_hat'] < 0,'Fatalities_hat'] = 0
df_val.loc[df_val['ConfirmedCases_hat'] < 0,'ConfirmedCases_hat'] = 0
df_val_3 = df_val.copy()
country1 = 'Luxembourg'
lat1 = df_geo[df_geo.Country_Region==country1]['Lat'].item()
lng1 = df_geo[df_geo.Country_Region==country1]['Long'].item()
country2 = 'Singapore'
lat2 = df_geo[df_geo.Country_Region==country2]['Lat'].item()
lng2 = df_geo[df_geo.Country_Region==country2]['Long'].item()
print('
Distance between ' + country1, country2, calc_distance(lat1, lng1, lat2, lng2))
print('This should be 10,436km')
 print(' Label Encoding the geographic features...')
 df1 = train[['Country_Region','Province_State','Lat','Long']].copy()
df2 = test[['Country_Region','Province_State','Lat','Long']].copy()
geo = pd.concat([df1,df2], axis=0)
 geo = geo.groupby(['Country_Region','Province_State'])[['Lat','Long']].max().reset_index()
le_country = LabelEncoder().fit(geo.Country_Region)
 le_state = LabelEncoder().fit(geo.Province_State)
 train['country'] = le_country.transform(train.Country_Region)
 train['state'] = le_state.transform(train.Province_State)
 test['country'] = le_country.transform(test.Country_Region)
 test['state'] = le_state.transform(test.Province_State)
 print('done', train.shape)
 train['wg'] = 0
train['wga'] = 0
train['wgr'] = 0
train['wgc'] = 0
train['wghcnb'] = 0
test['wg'] = 0
test['wga'] = 0
test['wgr'] = 0
test['wgc'] = 0
test['wghcnb'] = 0
test['SARIMAX'] = 0
test['ARIMA'] = 0
countries = ['Afghanistan']
for country in train.Country_Region.unique() :
bool1 = train.Country_Region == country
print(country)
 for state in train[bool1].Province_State.unique() :
bool2 = bool1 &(train.Province_State == state)&(train.ConfirmedCases>0)
 pop = np.max(train[bool2]['pop'])
 data = train[ bool2 ].copy().reset_index()
data['ts'] = data.index+1
dfj = data.iloc[0]['day_from_jan_first']
cdata = data['ConfirmedCases'].values
ts = data['ts'].values
boolt =(test.Country_Region==country)&(test.Province_State== state)
 datat = test[boolt].copy().reset_index()
datat['ts'] = datat.day_from_jan_first - dfj
datat.loc[datat.ts<=0,'ts'] = 1
x0 = [1343, 5.440110881178935, 2.188935325131958, 0.9897823619555628]
sol = minimize(wg_func, x0, constraints = cons)
 a,r,c,hcnb = sol.x[0], sol.x[1], sol.x[2], sol.x[3]
if np.isnan(sol.x[0]):
train.loc[bool2, 'wg'] = np.NaN
train.loc[bool2,'wga'] = np.NaN
train.loc[bool2,'wgr'] = np.NaN
train.loc[bool2,'wgc'] = np.NaN
train.loc[bool2,'wghcnb'] = np.NaN
test.loc[boolt, 'wg'] = np.NaN
test.loc[boolt,'wga'] = np.NaN
test.loc[boolt,'wgr'] = np.NaN
test.loc[boolt,'wgc'] = np.NaN
test.loc[boolt,'wghcnb'] = np.NaN
else:
datat['wg'] = datat.ts.apply(lambda x:(1-(a/(a+x**c)) **r)*pop*(1-hcnb)).values
data['wg'] = data.ts.apply(lambda x:(1-(a/(a+x**c)) **r)*pop*(1-hcnb)).values
train.loc[bool2, 'wg'] = data['wg'].values
train.loc[bool2,'wga'] = a
train.loc[bool2,'wgr'] = r
train.loc[bool2,'wgc'] = c
train.loc[bool2,'wghcnb'] = hcnb
test.loc[boolt, 'wg'] = datat.wg.values
test.loc[boolt,'wga'] = a
test.loc[boolt,'wgr'] = r
test.loc[boolt,'wgc'] = c
test.loc[boolt,'wghcnb'] = hcnb
incr = [cdata[i] if i == 0 else cdata[i] - cdata[i-1] for i,item in enumerate(cdata)]
cc = np.max(data[data.day_from_jan_first <=np.min(datat.day_from_jan_first)]['ConfirmedCases'])
 try:
model_arima = ARIMA(incr, order=(1,1,0)).fit()
preds = [item if item >=0 else 0 for item in model_arima.predict(datat.ts[0].item() , datat.ts[-1:].item())]
cum_sum = cc
preds_cc = []
for item in preds:
cum_sum = cum_sum + item
preds_cc.append(cum_sum)
 test.loc[boolt, 'ARIMA'] = pd.Series(preds_cc ).values
except:
test.loc[boolt, 'ARIMA']= np.NaN
try:
model_SARIMAX = SARIMAX(incr, order=(1,1,0), seasonal_order=(1,1,0,12),enforce_stationarity=False ).fit()
preds = [item if item >=0 else 0 for item in model_SARIMAX.predict(datat.ts[0].item() , datat.ts[-1:].item())]
cum_sum = cc
preds_cc = []
for item in preds:
cum_sum = cum_sum + item
preds_cc.append(cum_sum)
 test.loc[boolt, 'SARIMAX']= pd.Series(preds_cc ).values
except:
test.loc[boolt, 'SARIMAX']= np.NaN
print('done', datetime.now() )<feature_engineering>
sub1 = df_val_3
submission = sub1[['ForecastId','ConfirmedCases_hat','Fatalities_hat']]
submission.columns = ['ForecastId','ConfirmedCases','Fatalities']
train.loc[train.wg<0,'wg'] = 0
test.loc[test.wg<0,'wg'] = 0
train.loc[train.wg.isnull() ,'wg'] = 0
test.loc[test.wg.isnull() ,'wg'] = 0
train.loc[np.isinf(train.wg),'wg'] = 0
test.loc[np.isinf(test.wg),'wg'] = 0
print('done', datetime.now() )<feature_engineering>
TARGETS = ["ConfirmedCases", "Fatalities"]
sub_df = sub0.copy()
for t in TARGETS:
sub_df[t] = np.expm1(np.log1p(submission[t].values)*0.4 + np.log1p(sub0[t].values)*0.6)
 sub_df.to_csv("submission.csv", index=False )
<feature_engineering><EOS> 
sub0.isna().sum()