Kaggle competition: COVID19 Global Forecasting (Week 4)
val_perc_full = 0.12
n_valid_full = int(val_perc_full * len(train))
n_trn_full = len(train)-n_valid_full
df_full = train.drop(columns = ['winPlacePerc'])
 y = train['winPlacePerc']
df_full = df_full[to_keep]
X_train, X_valid = split_vals(df_full, n_trn_full)
 y_train, y_valid = split_vals(y, n_trn_full)
 print('Sample train shape: ', X_train.shape,
'Sample target shape: ', y_train.shape,
'Sample validation shape: ', X_valid.shape )<train_model>
train_df = pd.read_csv(PREFIX+"train.csv", parse_dates=["Date"])
 test_df = pd.read_csv(PREFIX+"test.csv", parse_dates=["Date"])
 sub_df = pd.read_csv(PREFIX+"submission.csv")
 LAG_DAYS = 28
MIN_DATE_TRAIN = train_df["Date"].min()
MID_DATE_TRAIN = train_df["Date"].min() + timedelta(days=LAG_DAYS)
 MAX_DATE_TRAIN = train_df["Date"].max()
START_DATE_TRAIN = MID_DATE_TRAIN + timedelta(days=1)
 MIN_DATE_TEST = test_df["Date"].min()
MAX_DATE_TEST = test_df["Date"].max()
print(MIN_DATE_TRAIN)
 print(MID_DATE_TRAIN)
 print(MAX_DATE_TRAIN)
 print(MIN_DATE_TEST)
 print(MAX_DATE_TEST)
 print('done')
 both_df = pd.concat([train_df,test_df[test_df["Date"]>MAX_DATE_TRAIN]],axis=0)
 print(train_df.shape)
 print(test_df.shape)
 print(both_df.shape)
 both_df.head(3)
 oof_df = train_df[train_df["Date"]<MID_DATE_TRAIN]
def create_oof_dict(oof_df):
first_death_list = []
first_case_list = []
country_list = oof_df["Country_Region"].unique().tolist()
for country in country_list:
df = oof_df[oof_df['Country_Region']==country]
df1 = df[df["ConfirmedCases"]>0]
if df1.shape[0] > 0:
first_case_list.append(df1["Date"].min())
 else:
first_case_list.append(MAX_DATE_TEST)
 df2 = df[df["Fatalities"]>0]
if df2.shape[0] > 0:
first_death_list.append(df2["Date"].min())
 else:
first_death_list.append(MAX_DATE_TEST)
 first_df = pd.DataFrame()
first_df['country'] = country_list
first_df['first_death_date'] = first_death_list
first_df['first_case_date'] = first_case_list
first_df['first_death_date'] =(first_df['first_death_date']-MIN_DATE_TRAIN ).dt.days
first_df['first_case_date'] =(first_df['first_case_date']-MIN_DATE_TRAIN ).dt.days
death_map = dict(zip(first_df['country'].tolist() ,first_df['first_death_date'].tolist()))
case_map = dict(zip(first_df['country'].tolist() ,first_df['first_case_date'].tolist()))
return case_map, death_map
case_map, death_map = create_oof_dict(oof_df)
 print("done")
 def replace_state_nan(df):
df = df.copy()
bool_index = df['Province_State'].isnull()
df.loc[bool_index, 'Province_State'] = df.loc[bool_index,'Country_Region'] + "_NaN"
return df
def create_days(df):
df = df.copy()
df['Days'] =(df["Date"]-MIN_DATE_TRAIN ).dt.days.astype(int)
 df['weekday'] = df["Date"].dt.dayofweek
return df
def create_CFR(df):
df = df.copy()
df["CFR"] = df["Fatalities"]/df["ConfirmedCases"]
df["CFR"].fillna(0, inplace=True)
 return df
korea_dummy_dict = defaultdict(int)
 korea_dummy_dict['Korea, South'] = 1
hubei_dummy_dict = defaultdict(int)
 hubei_dummy_dict['Hubei'] = 1
iran_dummy_dict = defaultdict(int)
 iran_dummy_dict['Iran'] = 1
us_dummy_dict = defaultdict(int)
 us_dummy_dict['US'] = 1
italy_dummy_dict = defaultdict(int)
 italy_dummy_dict['Italy'] = 1
COUNTRY_VC = train_df["Country_Region"].value_counts()
def hash_country(df):
df = df.copy()
df['country_hash'] = pd.util.hash_array(df['Country_Region'])
 df['country_hash'] = df['country_hash'].rank(pct=True)
 df['country_count'] = df['Country_Region'].map(COUNTRY_VC)
 df['state_hash'] = pd.util.hash_array(df['Province_State'])
 df['state_hash'] = df['Province_State'].rank(pct=True)
 df['first_death'] = df["Country_Region"].map(death_map)
 df['first_case'] = df["Country_Region"].map(case_map)
 df['dummy_korea'] = df["Country_Region"].map(korea_dummy_dict)
 df['dummy_hubei'] = df["Province_State"].map(hubei_dummy_dict)
 df['dummy_iran'] = df["Country_Region"].map(iran_dummy_dict)
 df['dummy_italy'] = df["Country_Region"].map(italy_dummy_dict)
 return df
def create_features(df):
df = replace_state_nan(df)
 df = hash_country(df)
 df = create_days(df)
 return df
both_df = create_features(both_df)
 both_df['state_date_key'] = both_df["Province_State"].astype(str)+ "_" + both_df["Date"].astype(str)
 case_dict = dict(zip(both_df['state_date_key'].tolist() , both_df['ConfirmedCases'].tolist()))
fatal_dict = dict(zip(both_df['state_date_key'].tolist() , both_df['Fatalities'].tolist()))
both_df.head(3)
 target_to_features_dict = {}
STATIC_FEATURES_LIST = ["country_hash","country_count","state_hash",
"first_death",
"first_case",
"Days","weekday",
"dummy_korea","dummy_hubei","dummy_iran"]
province_list = both_df["Province_State"].unique().tolist()
date_array = both_df[(both_df["Date"]>START_DATE_TRAIN)&(both_df["Date"]<=MAX_DATE_TRAIN)]["Date"].unique()
both_df["CFR"] = both_df["Fatalities"]/both_df["ConfirmedCases"]
both_df["CFR"].fillna(0, inplace=True)
 def update_target_dict(both_df, current_date, current_province_state):
features_list = []
target_df = both_df[(both_df["Province_State"]==current_province_state)&(both_df["Date"] == current_date)]
cases = target_df["ConfirmedCases"].values[0]
fatals = target_df["Fatalities"].values[0]
current_key = target_df["state_date_key"].values[0]
prior_df = both_df[(both_df["Province_State"]==current_province_state)& 
(both_df["Date"] < current_date)].copy()
prior_df.sort_values("Date",inplace=True)
 prior_df = prior_df.tail(LAG_DAYS ).copy()
prior_df.reset_index(drop=True, inplace=True)
 features_list += [prior_df["Fatalities"].max() ]
features_list += [prior_df["ConfirmedCases"].max() ]
features_list += [prior_df["CFR"].max() ]
features_list += [(prior_df["Fatalities"]*prior_df["ConfirmedCases"] ).max() ]
features_list += [prior_df["Fatalities"].diff().abs().max() ]
features_list += [prior_df["ConfirmedCases"].diff().abs().max() ]
features_list += [prior_df["CFR"].diff().abs().max() ]
features_list += target_df[STATIC_FEATURES_LIST].values.tolist() [0]
for f in ["ConfirmedCases","Fatalities","CFR"]:
features_list += prior_df[f].tolist()
for f in ["ConfirmedCases","Fatalities"]:
features_list +=(prior_df[f].diff(2)/(1+prior_df[f])).dropna().tolist()
features_list +=(-prior_df["ConfirmedCases"]+prior_df["Fatalities"] ).tolist()
model = make_pipeline(PolynomialFeatures(2), BayesianRidge())
 model.fit(prior_df[["ConfirmedCases","Fatalities","Days"]].shift(periods=2 ).dropna() ,
prior_df[["ConfirmedCases"]].shift(periods=-2 ).dropna().values.reshape(-1,))
preds = model.predict(prior_df[["ConfirmedCases","Fatalities","Days"]])
 features_list += [preds[-1]]
model.fit(prior_df[["ConfirmedCases","Fatalities","Days"]].shift(periods=2 ).dropna() ,
prior_df[["Fatalities"]].shift(periods=-2 ).dropna().values.reshape(-1,))
preds = model.predict(prior_df[["ConfirmedCases","Fatalities","Days"]])
 features_list += [preds[-1]]
model = make_pipeline(PolynomialFeatures(2), BayesianRidge())
 model.fit(prior_df[["ConfirmedCases","Fatalities","Days"]].diff(periods=1 ).dropna() ,
prior_df[["ConfirmedCases"]].diff(periods=-1 ).dropna().values.reshape(-1,))
preds = model.predict(prior_df[["ConfirmedCases","Fatalities","Days"]].diff(1 ).dropna())
 features_list += [preds[-1]]
model.fit(prior_df[["ConfirmedCases","Fatalities","Days"]].diff(periods=1 ).dropna() ,
prior_df[["Fatalities"]].diff(periods=-1 ).dropna().values.reshape(-1,))
preds = model.predict(prior_df[["ConfirmedCases","Fatalities","Days"]].diff(1 ).dropna())
 features_list += [preds[-1]]
features_list += [(prior_df["Days"]-prior_df["first_death"] ).max() ]
features_list += [(prior_df["Days"]-prior_df["first_case"] ).max() ]
target_to_features_dict[(current_province_state, current_date, current_key)] = [cases,fatals,features_list]
for current_province_state in province_list:
for current_date in date_array:
update_target_dict(both_df, current_date, current_province_state)
 print("done")
 def list_to_matrix(x):
return np.array([np.array(xi)for xi in x])
 def create_train_X_y(target_to_features_dict):
X_train = []
y_key = []
y_train_case = []
y_train_fatal = []
for k,v in target_to_features_dict.items() :
X_train.append(v[2])
 y_train_case.append(v[0])
 y_train_fatal.append(v[1])
 y_key.append(k[2])
 X_train = list_to_matrix(X_train)
 return X_train, y_train_case, y_train_fatal, y_key
X_train, y_train_case, y_train_fatal, y_key = create_train_X_y(target_to_features_dict)
 print(X_train.shape)
 print("done")
 def create_test_X_y(target_to_features_dict, date_array_0):
X_test = []
y_key = []
y_test_case = []
y_test_fatal = []
for k,v in target_to_features_dict.items() :
if k[1] == date_array_0:
X_test.append(v[2])
 y_test_case.append(v[0])
 y_test_fatal.append(v[1])
 y_key.append(k[2])
 X_test = list_to_matrix(X_test)
 return X_test, y_test_case, y_test_fatal, y_key
np.random.seed(2)
 def rmsle(y_true, y_pred):
return 'RMSLE', np.sqrt(np.mean(np.power(np.log1p(y_pred)- np.log1p(y_true), 2))), False
num_features = X_train.shape[1]
mc = ["0"]*(num_features-4)
 MC_STR = "(1,1,1,1," + ",".join(mc)+ ")"
def xgb_reg() :
return XGBRegressor(n_estimators=2000,
booster="gbtree",
tree_method = "exact",
eta= 0.003,
max_depth=6,
subsample=0.9,
colsample_bytree=0.9,
colsample_bylevel=0.9,
min_child_weight = 0,
objective = "reg:squarederror",
reg_alpha=0.1,
reg_lambda=0.95)
 def dual_xgb_fit(X_train, y_train_case, y_train_fatal):
gbm_case = xgb_reg()
gbm_case.fit(X_train, np.log1p(y_train_case))
gbm_fatal = xgb_reg()
gbm_fatal.fit(X_train, np.log1p(y_train_fatal))
return gbm_case, gbm_fatal
gbm_case, gbm_fatal = dual_xgb_fit(X_train, y_train_case, y_train_fatal)
 print("done")
 print(both_df["state_date_key"].nunique())
 print(both_df.shape)
 print(both_df["Date"][both_df["Fatalities"].isnull() ].min())
 date_array = [both_df["Date"][both_df["Fatalities"].isnull() ].min() ]
print(date_array)
 both_df["CFR"] = both_df["Fatalities"]/both_df["ConfirmedCases"]
both_df["CFR"].fillna(0, inplace=True)
 for current_province_state in province_list:
for current_date in date_array:
update_target_dict(both_df, current_date, current_province_state)
 print("done")
 X_test, y_test_case, y_test_fatal, y_key = create_test_X_y(target_to_features_dict, date_array[0])
 print(X_test.shape)
 y_pred_case = np.expm1(gbm_case.predict(X_test))
y_pred_fatal = np.expm1(gbm_fatal.predict(X_test))
print("done")
 for j in range(len(y_key)) :
key_val = y_key[j]
case_dict[key_val] = y_pred_case[j]
fatal_dict[key_val] = y_pred_fatal[j]
print("done")
 both_df["ConfirmedCases"] = both_df["state_date_key"].map(case_dict)
 both_df["Fatalities"] = both_df["state_date_key"].map(fatal_dict)
 both_df["CFR"] = both_df["Fatalities"]/both_df["ConfirmedCases"]
both_df["CFR"].fillna(0, inplace=True)
 print("done")
 print(date_array)
 both_df["CFR"] = both_df["Fatalities"]/both_df["ConfirmedCases"]
both_df["CFR"].fillna(0, inplace=True)
 for current_province_state in province_list:
for current_date in date_array:
update_target_dict(both_df, current_date, current_province_state)
 print("done")
 X_train, y_train_case, y_train_fatal, y_key = create_train_X_y(target_to_features_dict)
 print(X_train.shape)
 print("done")
 gbm_case, gbm_fatal = dual_xgb_fit(X_train, y_train_case, y_train_fatal)
 print("done")
 date_array = [both_df["Date"][both_df["Fatalities"].isnull() ].min() ]
print(date_array)
 both_df["CFR"] = both_df["Fatalities"]/both_df["ConfirmedCases"]
both_df["CFR"].fillna(0, inplace=True)
 for current_province_state in province_list:
for current_date in date_array:
update_target_dict(both_df, current_date, current_province_state)
 print("done")
 X_test, y_test_case, y_test_fatal, y_key = create_test_X_y(target_to_features_dict, date_array[0])
 print(X_test.shape)
 y_pred_case = np.expm1(gbm_case.predict(X_test))
y_pred_fatal = np.expm1(gbm_fatal.predict(X_test))
print("done")
 for j in range(len(y_key)) :
key_val = y_key[j]
case_dict[key_val] = y_pred_case[j]
fatal_dict[key_val] = y_pred_fatal[j]
print(len(case_dict.keys()))
print("done")
 both_df["ConfirmedCases"] = both_df["state_date_key"].map(case_dict)
 both_df["Fatalities"] = both_df["state_date_key"].map(fatal_dict)
 print("done")
 null_count = both_df["Fatalities"].isnull().sum()
print(null_count)
 while null_count > 0:
print(null_count)
 both_df["CFR"] = both_df["Fatalities"]/both_df["ConfirmedCases"]
both_df["CFR"].fillna(0, inplace=True)
 for current_province_state in province_list:
for current_date in date_array:
update_target_dict(both_df, current_date, current_province_state)
 X_train, y_train_case, y_train_fatal, y_key = create_train_X_y(target_to_features_dict)
 gbm_case, gbm_fatal = dual_xgb_fit(X_train, y_train_case, y_train_fatal)
 date_array = [both_df["Date"][both_df["Fatalities"].isnull() ].min() ]
print(date_array)
 for current_province_state in province_list:
for current_date in date_array:
update_target_dict(both_df, current_date, current_province_state)
 X_test, y_test_case, y_test_fatal, y_key = create_test_X_y(target_to_features_dict, date_array[0])
 print(X_test.shape)
 y_pred_case = np.expm1(gbm_case.predict(X_test))
y_pred_fatal = np.expm1(gbm_fatal.predict(X_test))
for j in range(len(y_key)) :
key_val = y_key[j]
case_dict[key_val] = y_pred_case[j]
fatal_dict[key_val] = y_pred_fatal[j]
print(len(case_dict.keys()))
both_df["ConfirmedCases"] = both_df["state_date_key"].map(case_dict)
 both_df["Fatalities"] = both_df["state_date_key"].map(fatal_dict)
 null_count = both_df["Fatalities"].isnull().sum()
print("done")
 both_df.tail(10)
 test_df = pd.read_csv(PREFIX+"test.csv", parse_dates=["Date"])
 test_df = replace_state_nan(test_df)
 test_df.head(3)
 print(test_df.shape)
 merge_df = both_df[["Province_State","Date","ConfirmedCases","Fatalities"]].copy()
merge_df.sort_values(["Province_State","Date"], inplace=True)
 print(merge_df.head(5))
merge_df["ConfirmedCases"] = merge_df.groupby('Province_State')["ConfirmedCases"].cummax()
merge_df["Fatalities"] = merge_df.groupby('Province_State')["Fatalities"].cummax()
test_df = test_df.merge(merge_df, how="inner",
on=["Province_State","Date"])
 print(test_df.shape)
 test_df[["ForecastId","ConfirmedCases","Fatalities"]].to_csv("submission.csv", index=False)
 print("done")