Kaggle competition: COVID19 Global Forecasting (Week 4)
<SOS> metric: MCRMSLE Kaggle data source: covid19-global-forecasting-(week-4)<categorify>
import os
bin_map = {'TA':2,'Gd':3, 'Fa':1,'Ex':4,'Po':1,'None':0,'Y':1,'N':0,'Reg':3,'IR1':2,'IR2':1,'IR3':0,
"No" : 2, "Mn" : 2, "Av": 3,"Gd" : 4,"Unf" : 1, "LwQ": 2, "Rec" : 3,"BLQ" : 4, "ALQ" : 5, "GLQ" : 6,
'Typ':7, 'Min1':6, "Min2":5, 'Mod':4, "Maj1":3, "Maj2":2, "Sev":1, "Sal":0,'Grvl':0,'Pave':1
}
train_test['ExterQual'] = train_test['ExterQual'].map(bin_map)
 train_test['ExterCond'] = train_test['ExterCond'].map(bin_map)
 train_test['BsmtCond'] = train_test['BsmtCond'].map(bin_map)
 train_test['BsmtQual'] = train_test['BsmtQual'].map(bin_map)
 train_test['HeatingQC'] = train_test['HeatingQC'].map(bin_map)
 train_test['KitchenQual'] = train_test['KitchenQual'].map(bin_map)
 train_test['FireplaceQu'] = train_test['FireplaceQu'].map(bin_map)
 train_test['Functional'] = train_test['Functional'].map(bin_map)
 train_test['Street'] = train_test['Street'].map(bin_map)
 train_test['GarageQual'] = train_test['GarageQual'].map(bin_map)
 train_test['GarageCond'] = train_test['GarageCond'].map(bin_map)
 train_test['CentralAir'] = train_test['CentralAir'].map(bin_map)
 train_test['LotShape'] = train_test['LotShape'].map(bin_map)
 train_test['BsmtExposure'] = train_test['BsmtExposure'].map(bin_map)
 train_test['BsmtFinType1'] = train_test['BsmtFinType1'].map(bin_map)
 train_test['BsmtFinType2'] = train_test['BsmtFinType2'].map(bin_map)
 PavedDrive = {"N" : 0, "P" : 1, "Y" : 2}
train_test['PavedDrive'] = train_test['PavedDrive'].map(PavedDrive)
 GarageFinish = {'Fin':3, 'RFn':2, 'Unf':1, 'NA':0,'None':0}
train_test['GarageFinish'] = train_test['GarageFinish'].map(GarageFinish)
 LandSlope = {'Gtl':2, 'Mod':1, 'Sev':0}
train_test['LandSlope'] = train_test['LandSlope'].map(LandSlope)
 train_test["MSZoning"] = train_test.MSZoning.map({'C(all)':1, 'RH':2, 'RM':2, 'RL':3, 'FV':4})
 train_test["oCondition1"] = train_test.Condition1.map({'Artery':1,
'Feedr':2, 'RRAe':2,
'Norm':3, 'RRAn':3,
'PosN':4, 'RRNe':4,
'PosA':5 ,'RRNn':5} )<feature_engineering>
%matplotlib inline
train_test['Total_SF']=train_test['TotalBsmtSF'] + train_test['1stFlrSF'] + train_test['2ndFlrSF']
train_test['Total_sqr_footage'] =(train_test['BsmtFinSF1'] + train_test['BsmtFinSF2'] +
train_test['1stFlrSF'] + train_test['2ndFlrSF'])
 train_test['Total_Bathrooms'] =(train_test['FullBath'] +(0.5 * train_test['HalfBath'])+
train_test['BsmtFullBath'] +(0.5 * train_test['BsmtHalfBath']))
train_test['Total_porch_sf'] =(train_test['OpenPorchSF'] + train_test['3SsnPorch'] +
train_test['EnclosedPorch'] + train_test['ScreenPorch'] +
train_test['WoodDeckSF'])
 train_test["+_TotalHouse_OverallQual"] = train_test["Total_SF"] * train_test["OverallQual"]
train_test["+_GrLivArea_OverallQual"] = train_test["GrLivArea"] * train_test["OverallQual"]
train_test["+_oMSZoning_TotalHouse"] = train_test["MSZoning"] * train_test["Total_SF"]
train_test["+_oMSZoning_OverallQual"] = train_test["MSZoning"] + train_test["OverallQual"]
train_test["+_oNeighborhood_TotalHouse"] = train_test["oNeighborhood"] * train_test["Total_SF"]
train_test["+_oNeighborhood_OverallQual"] = train_test["oNeighborhood"] + train_test["OverallQual"]
train_test["+_oNeighborhood_YearBuilt"] = train_test["oNeighborhood"] + train_test["YearBuilt"]
train_test["+_BsmtFinSF1_OverallQual"] = train_test["BsmtFinSF1"] * train_test["OverallQual"]
train_test["-_oFunctional_TotalHouse"] = train_test["Functional"] * train_test["Total_SF"]
train_test["-_oFunctional_OverallQual"] = train_test["Functional"] + train_test["OverallQual"]
train_test["-_LotArea_OverallQual"] = train_test["LotArea"] * train_test["OverallQual"]
train_test["-_TotalHouse_LotArea"] = train_test["Total_SF"] + train_test["LotArea"]
train_test["-_oCondition1_TotalHouse"] = train_test["oCondition1"] * train_test["Total_SF"]
train_test["-_oCondition1_OverallQual"] = train_test["oCondition1"] + train_test["OverallQual"]
train_test["Bsmt"] = train_test["BsmtFinSF1"] + train_test["BsmtFinSF2"] + train_test["BsmtUnfSF"]
train_test["Rooms"] = train_test["FullBath"]+train_test["TotRmsAbvGrd"]
train_test["TotalPlace"] = train_test["TotalBsmtSF"] + train_test["1stFlrSF"] + train_test["2ndFlrSF"] + train_test["GarageArea"] +train_test["OpenPorchSF"]+train_test["EnclosedPorch"]+train_test["3SsnPorch"]+train_test["ScreenPorch"]<create_dataframe>
import lightgbm as lgb
train_test_num = pd.DataFrame(train_test.select_dtypes(['float64','int32','int64']))
train_test_num.drop(columns=ordinal_features,inplace=True)
 train_test_num.drop(columns=binary_features,inplace=True )<create_dataframe>
from tqdm.notebook import tqdm
train_test_skewed = train_test_num.apply(lambda x:skew(x)).sort_values(key=abs,ascending=False)
 train_test_skewed_df = pd.DataFrame({'Skew':train_test_skewed})
 train_test_skewed_df<feature_engineering>
from sklearn.preprocessing import LabelEncoder
skewed_features = train_test_skewed_df[train_test_skewed_df.Skew>0.5].index
lmbd = 0.15
for sk in skewed_features:
if sk not in ['SalePrice','SalePrice_log']:
train_test[sk] = boxcox1p(train_test[sk], lmbd )<count_missing_values>
train_df = pd.read_csv(".. /input/covid19-global-forecasting-week-4/train.csv")
 test_df = pd.read_csv(".. /input/covid19-global-forecasting-week-4/test.csv")
 submission_df = pd.read_csv(".. /input/covid19-global-forecasting-week-4/submission.csv" )
train_test = pd.get_dummies(train_test)
 train_test.isna().sum().sort_values(ascending=False )<train_model>
region_metadata = pd.read_csv(".. /input/covid19-forecasting-metadata/region_metadata.csv" )
lasso_model = Lasso(alpha=0.0001 ).fit(train_test[:1458],train.SalePrice_log)
 ft_im_df = pd.DataFrame({"Feature Importance":lasso_model.coef_}, index=train_test.columns)
 ft_im_df.sort_values('Feature Importance', key=abs,ascending=False ).head(20 )<normalization>
def transform_geo_location(df):
lat = df.lat.values
lon = df.lon.values
df["lat_sin"] = np.sin(2 * np.pi * lat / 360)
 df["lat_cos"] = np.cos(2 * np.pi * lat / 360)
 df["lon_sin"] = np.sin(2 * np.pi * lon / 360)
 df["lon_cos"] = np.cos(2 * np.pi * lon / 360)
 return df
CV = KFold(n_splits=5,shuffle=True,random_state=42)
 rb_scaler = RobustScaler()<compute_train_metric>
region_metadata = transform_geo_location(region_metadata)
 geo_location_columns = ["lat_sin", "lat_cos", "lon_sin", "lon_cos"]
def rmse(model,x,y):
return -(cross_val_score(model, x, y,cv=CV,scoring='neg_root_mean_squared_error',n_jobs=-1))<prepare_x_and_y>
country_data = pd.read_csv(".. /input/countries-of-the-world/countries of the world.csv")
 country_data.Country = country_data.Country.apply(lambda c: c.rstrip(" "))
country_data.rename(
 columns={
"Country": "Country_Region",
},
inplace=True
)
train_test = rb_scaler.fit_transform(train_test)
 X = train_test[:1458]
X_test = train_test[1458:]
Y = train.SalePrice_log
X.shape<categorify>
country_meta_columns = [
'Coastline(coast/area ratio)',
'Net migration',
'Infant mortality(per 1000 births)',
'GDP($ per capita)',
'Literacy(%)',
'Phones(per 1000)',
'Arable(%)',
'Crops(%)',
'Other(%)',
'Climate',
'Birthrate',
'Deathrate',
'Agriculture',
'Industry',
'Service'
]
pca = PCA(n_components=0.9999)
 train_test_pc = pca.fit_transform(train_test)
 X_PC = train_test_pc[:1458]
X_test_PC = train_test_pc[1458:]
X_PC.shape<define_variables>
for col in country_meta_columns:
country_data[col] =(
 country_data[col] 
.fillna(-1000)
.apply(lambda v: v.replace(",", ".")if isinstance(v, str)else v)
.astype(np.float32)
 )
n_trials = 500<choose_model_class>
lockdown_meta_df = pd.read_csv(".. /input/covid19-lockdown-dates-by-country/countryLockdowndates.csv")
 lockdown_meta_df.rename(
 columns={
"Country/Region": "Country_Region",
"Province": "Province_State",
"Date": "LockdownDate",
"Type": "LockdownType"
},
inplace=True 
)
 lockdown_meta_df.drop("Reference", axis=1, inplace=True)
 def convert_lockdown_date(d):
if isinstance(d, str):
return "-".join(reversed(d.split("/")))
 else:
return "2020-12-31"
lockdown_meta_df.LockdownDate = lockdown_meta_df.LockdownDate.apply(convert_lockdown_date )
ridge_best_params = {'alpha': 36.31161037749121}
ridge_best_model = Ridge(**ridge_best_params )<choose_model_class>
train_df = train_df.merge(lockdown_meta_df, on=["Province_State", "Country_Region"], how="left")
 train_df = train_df.merge(country_data, on=["Country_Region"], how="left" )
lasso_best_params = {'alpha': 0.0007350225758112973}
lasso_best_model = Lasso(**lasso_best_params )<choose_model_class>
train_df["days_since_lockdown"] = np.clip(
(pd.to_datetime(train_df.Date)- pd.to_datetime(train_df.LockdownDate)).dt.days,
a_min=-1,
a_max=None 
)
 train_df["lockdown_type"] = [
t if d >= 0 else "None" for t, d in zip(train_df.LockdownType, train_df.days_since_lockdown)
 ]
lockdown_type_encoder = LabelEncoder().fit(train_df.lockdown_type)
 train_df["lockdown_type"] = lockdown_type_encoder.transform(train_df.lockdown_type )
elastic_best_params = {'alpha': 0.004962477837834253, 'l1_ratio': 0.1}
elastic_best_model = ElasticNet(**elastic_best_params )<choose_model_class>
train_df["ConfirmedCases"] = np.log1p(train_df.ConfirmedCases)
 train_df["Fatalities"] = np.log1p(train_df.Fatalities )
SVR_best_params = {'epsilon': 0.0007028077160999125, 'kernel': 'rbf',
'gamma': 0.0001032023906441773, 'C': 20.673761351762828}
SVR_best_model = SVR(**SVR_best_params )<choose_model_class>
def extract_region(df):
return(
 df.Country_Region +
df.Province_State.fillna("" ).apply(lambda s: " + " + s if s else s)
)
 region_encoder = LabelEncoder().fit(extract_region(train_df))
train_df["region_id"] = region_encoder.transform(extract_region(train_df))
test_df["region_id"] = region_encoder.transform(extract_region(test_df))
region_metadata["region_id"] = region_encoder.transform(extract_region(region_metadata))
region_metadata.drop(["Province_State", "Country_Region"], axis=1, inplace=True )
KNR_best_params = {'alpha': 0.017852963690058583, 'kernel': 'polynomial',
'gamma': 0.00010010824404031089, 'degree': 4}
KNR_best_model = KernelRidge(**KNR_best_params )<choose_model_class>
stats_df = train_df[["Fatalities", "ConfirmedCases", "region_id"]].groupby("region_id" ).sum()
stats_df["fatalities_to_cases"] = stats_df.Fatalities - stats_df.ConfirmedCases
stats_df.drop(["Fatalities", "ConfirmedCases"], axis=1, inplace=True)
 train_df = train_df.merge(stats_df, on="region_id", how="left")
 stats_columns = ["fatalities_to_cases"]
Bayesian_best_params = {'alpha_1': 9.974811163419825e-06, 'alpha_2': 5.050535825995536e-06,
'lambda_1': 9.97006607898247e-06, 'lambda_2': 1.004807953151607e-08}
Bayesian_best_model = BayesianRidge(**Bayesian_best_params )<choose_model_class>
def extract_country_from_region(region):
if "+" in region:
country, state = region.split(" + ")
 else:
country = region
return country
region_to_country = dict(
 zip(
 range(len(region_encoder.classes_)) ,
map(extract_country_from_region, region_encoder.classes_)
)
 )
<choose_model_class>
train_df = train_df.merge(region_metadata, on="region_id", how="left" )
MLP_best_model = MLPRegressor(random_state=42,
solver='adam',
activation='tanh',
alpha= 2.9344049246725894,
hidden_layer_sizes=(120),
max_iter=1000,
learning_rate='adaptive',
batch_size=300,
learning_rate_init=0.5,
early_stopping=True )<init_hyperparams>
meta_columns = [
"density",
"population",
"area"
]
<choose_model_class>
categorical_columns = [
"region_id",
"lockdown_type"
]
XGB_best_params= {'n_estimators': 230, 'learning_rate': 0.05064350395549183,
'max_depth': 2, 'booster': 'gblinear', 'gamma': 0.0005562582585578549,
'reg_alpha': 0.0010013453661588325, 'reg_lambda': 0.0010065125775610678,
'colsample_bytree': 0.6000000000000001, 'subsample': 1.0}
XGB_best_model = XGBRegressor(**XGB_best_params,objective='reg:squaredlogerror',
random_state=42,n_jobs=-1 )<init_hyperparams>
def compute_historical_features(df, target_column, past_horizon, create_target=True):
features = dict()
for col in meta_columns + country_meta_columns:
features[col] = df[col].unique().item()
for col in geo_location_columns:
features[col] = df[col].unique().item()
for col in stats_columns:
features[col] = df[col].unique().item()
features["region_id"] = df["region_id"].unique().item()
last = -1 if create_target else len(df)
 target_values = df[target_column].values
for lag in range(past_horizon):
features["value_{}_{}".format(target_column, lag)] = target_values[last-lag-1]
features["days_since_lockdown"] = df.days_since_lockdown.max()
features["lockdown_type"] = df.lockdown_type.values[last-1]
if create_target:
features[target_column] = df[target_column].values[-1]
return features
<init_hyperparams>
def prepare_datasets(
 train_df,
last_train_date,
target_column,
train_start_offset=50,
step=1, 
):
dates = train_df.Date.unique()
train_dates = dates[dates <= last_train_date]
val_dates = dates[dates > last_train_date]
train_features = []
train_subdf = train_df[train_df.Date <= last_train_date]
val_subdf = train_df[train_df.Date > last_train_date]
for region_id in tqdm(sorted(train_df.region_id.unique()), desc="Making train features"):
subdf = train_subdf[train_subdf.region_id == region_id]
for date in train_dates[train_start_offset::step]:
features = compute_historical_features(
 subdf[subdf.Date <= date],
target_column=target_column,
past_horizon=train_start_offset,
create_target=True 
)
 train_features.append(features)
 return pd.DataFrame(train_features), train_subdf, val_subdf
LGBM_best_params={'boosting_type': 'gbdt', 'num_leaves': 10,
'learning_rate': 0.0623344941102757, 'n_estimators': 350,
'max_depth': 6, 'min_samples_split': 0.003721770553125834,
'min_samples_leaf': 2, 'reg_alpha': 0.011733455705754578,
'reg_lambda': 0.0692034942836032, 'colsample_bytree': 0.6000000000000001,
'subsample': 0.9}
LGBM_best_model = LGBMRegressor(**LGBM_best_params,random_state=42,n_jobs=-1 )<choose_model_class>
def autoregressive_predict(model, dates, df, target_column, past_horizon):
n_regions = len(region_encoder.classes_)
 original_size = len(df)
 future_df = pd.DataFrame({
"Date": np.repeat(dates, repeats=n_regions),
"region_id": np.tile(np.arange(n_regions), len(dates))
})
 future_df["Country_Region"] = future_df.region_id.map(region_to_country)
 combined_df = pd.concat([
df.drop(meta_columns + country_meta_columns + geo_location_columns + stats_columns, axis=1),
future_df
], sort=True)
 combined_df = combined_df.merge(region_metadata, on="region_id", how="left")
 combined_df = combined_df.merge(country_data, on=["Country_Region"], how="left")
 combined_df = combined_df.merge(stats_df, on="region_id", how="left")
 for date in tqdm(dates, "Prediction"):
date_features = []
for region_id in range(len(region_encoder.classes_)) :
cond =(combined_df.Date < date)&(combined_df.region_id == region_id)
 features = compute_historical_features(
 combined_df[cond],
target_column=target_column,
past_horizon=past_horizon,
create_target=False 
)
 date_features.append(features)
 date_predictions = model.predict(pd.DataFrame(date_features))
combined_df.loc[combined_df.Date == date, target_column] = date_predictions
return combined_df[["Date", "region_id", target_column]].loc[original_size:].reset_index(drop=True )
def model_eval() :
rmse_mean = []
std = []
rmser = []
regressors=['Lasso','Ridge','Elastic_Net',
'Kernel_Ridge','SVR','Bayesian',
'MLP','XGB','LGBM']
models=[lasso_best_model,ridge_best_model,
elastic_best_model,KNR_best_model,
SVR_best_model,Bayesian_best_model,
MLP_best_model,XGB_best_model,
LGBM_best_model]
for model in models:
cv_result = rmse(model,X_PC,Y)
 rmse_mean.append(cv_result.mean())
 std.append(cv_result.std())
 rmser.append(cv_result)
 performance_df=pd.DataFrame({'CV_Mean':rmse_mean,'Std':std},index=regressors)
 return performance_df
performance_df = model_eval()
performance_df.sort_values('CV_Mean' )<choose_model_class>
def target_metric(actual, predicted):
return np.sqrt(((actual - predicted)** 2 ).mean() )
stacking = StackingRegressor(estimators=[('SVR',SVR_best_model), 
('Kernel_ridge',KNR_best_model), 
('ElasticNet',elastic_best_model), 
('Ridge',ridge_best_model), 
('Lasso', lasso_best_model), 
('Bayesian',Bayesian_best_model), 
('MLP',MLP_best_model), 
('LGBM',LGBM_best_model), 
('XGB',XGB_best_model)],
final_estimator=KernelRidge() ,
cv=5,
n_jobs=-1)
 cv_result = rmse(stacking,X_PC,Y)
 stk_acc = cv_result.mean()
stk_std = cv_result.std()
performance_df.loc['Stacking'] = {'CV_Mean':stk_acc, 'Std':stk_std}
performance_df.sort_values(by=['CV_Mean'] )<find_best_model_class>
def train_booster(
 train_features_df,
target_column,
categorical_columns,
epochs=20,
train_start_offset=50,
val_df=None 
):
train_dataset = lgb.Dataset(
 train_features_df.drop([target_column], axis=1),
train_features_df[target_column],
free_raw_data=False,
categorical_feature=categorical_columns 
)
 params = {
'boosting_type': 'gbdt',
'objective': 'rmse',
'num_leaves': 31,
'max_depth': 7,
'learning_rate': 0.01,
'feature_fraction': 0.5,
'num_threads': os.cpu_count()
}
rounds_per_epoch = 300
model = None
for epoch in range(epochs):
model = lgb.train(
 params,
train_dataset,
num_boost_round=rounds_per_epoch,
valid_sets=[train_dataset],
valid_names=["train"],
verbose_eval=rounds_per_epoch // 5,
init_model=model 
)
 if val_df is not None:
val_dates = val_df.Date.unique()
val_predictions = autoregressive_predict(
 model,
dates=val_dates,
df=train_subdf,
past_horizon=train_start_offset,
target_column=target_column 
)
 error = target_metric(
 val_subdf.sort_values(by=["Date", "region_id"])[target_column].values,
val_predictions.sort_values(by=["Date", "region_id"])[target_column].values 
)
 print("Epoch", epoch, "error:", error)
 print()
return model
stack_y = stacking.fit(X_PC,Y ).predict(X_test_PC)
 elastic_y = elastic_best_model.fit(X_PC,Y ).predict(X_test_PC)
 ridge_y = ridge_best_model.fit(X_PC,Y ).predict(X_test_PC)
 lasso_y = lasso_best_model.fit(X_PC,Y ).predict(X_test_PC)
 SVR_y = SVR_best_model.fit(X_PC,Y ).predict(X_test_PC)
 KNR_y = KNR_best_model.fit(X_PC,Y ).predict(X_test_PC)
 Bayesian_y = Bayesian_best_model.fit(X_PC,Y ).predict(X_test_PC)
 XGB_y = XGB_best_model.fit(X_PC,Y ).predict(X_test_PC)
 blend = 0.0 * stack_y + 0.1 * elastic_y + 0.9*ridge_y + 0.0*lasso_y+ 0.0*SVR_y 
<feature_engineering>
train_start_offset = 70
epochs = 10
test_dates = test_df.Date[test_df.Date > train_df.Date.max() ].unique()
merge_columns = ["Date", "region_id"]
for target_column in ["ConfirmedCases", "Fatalities"]:
print()
print("Working on column", target_column)
 print()
train_features_df, train_subdf, val_subdf = prepare_datasets(
 train_df,
last_train_date=train_df.Date.max() ,
target_column=target_column,
train_start_offset=train_start_offset 
)
 model = train_booster(
 train_features_df,
target_column=target_column,
categorical_columns=categorical_columns,
epochs=epochs,
train_start_offset=train_start_offset,
val_df=None 
)
 test_predictions = autoregressive_predict(
 model,
dates=test_dates,
df=train_subdf,
past_horizon=train_start_offset,
target_column=target_column 
)
 predicted_test_df = test_df.merge(
 pd.concat([
train_df[merge_columns + [target_column]],
test_predictions[merge_columns + [target_column]]
], sort=True),
on=merge_columns,
how="left" 
)
 submission_df =(
 submission_df 
.drop([target_column], axis=1)
.merge(
 predicted_test_df[[target_column, "ForecastId"]],
on=["ForecastId"],
how="left" 
)
)
 submission_df[target_column] = np.expm1(submission_df[target_column] )
<save_to_csv><EOS> 
submission_df.to_csv("submission.csv", index=False )