Kaggle competition: COVID19 Global Forecasting (Week 4)
<SOS> metric: MCRMSLE Kaggle data source: covid19-global-forecasting-(week-4)<groupby>
import numpy as np
import pandas as pd
import seaborn as sns
from sklearn.model_selection import train_test_split
from xgboost import XGBRegressor
from sklearn.multioutput import MultiOutputRegressor
from sklearn.impute import SimpleImputer
meanMSSubClass_Saleprice = Train[['MSSubClass', 'SalePrice_log']].groupby('MSSubClass' ).mean()<feature_engineering>
train_data = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/train.csv')
 test_data = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/test.csv')
 submission_csv = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/submission.csv' )
for col in Test.columns:
if len(set(Train[col])) > 2 and '_log' not in col:
Train[f'{col}_log'] = np.log(Train[col] + 1)
 Train = Train.drop(col, axis = 1)
 Test[f'{col}_log'] = np.log(Test[col] + 1)
 Test = Test.drop(col, axis = 1 )<feature_engineering>
train_data['Date'] = pd.to_datetime(train_data['Date'], infer_datetime_format=True)
 test_data['Date'] = pd.to_datetime(test_data['Date'], infer_datetime_format=True )
<drop_column>
train_data.loc[:, 'Date'] = train_data.Date.dt.strftime('%y%m%d')
 train_data.loc[:, 'Date'] = train_data['Date'].astype(int)
 test_data.loc[:, 'Date'] = test_data.Date.dt.strftime('%y%m%d')
 test_data.loc[:, 'Date'] = test_data['Date'].astype(int )
Train = Train.drop(1276 )<sort_values>
train_data['Province_State'] = np.where(train_data['Province_State'] == 'nan',train_data['Country_Region'],train_data['Province_State'])
 test_data['Province_State'] = np.where(test_data['Province_State'] == 'nan',test_data['Country_Region'],test_data['Province_State'] )
corr = Train.corr() ['SalePrice_log']
Train_err = Train[corr[np.abs(corr)< 0.4].index]
Test_err = Test[corr[np.abs(corr)< 0.4].index]
Train = Train[corr[np.abs(corr)> 0.4].index]
Test = Test[corr[np.abs(corr)> 0.4].index.drop('SalePrice_log')] 
<define_variables>
convert_dict = {'Province_State': str}
train_data = train_data.astype(convert_dict)
 test_data = test_data.astype(convert_dict )
def getFoldValidation(length, K = 4):
diff = int(length/K)
 fraction = length % K
batchValidation = [[k*diff,(k + 1)*diff] for k in range(K)]
batchValidation[-1][1] = batchValidation[-1][1] + fraction - 1
return batchValidation
batchValidation = np.array(getFoldValidation(len(Train)) )<categorify>
s =(train_data.dtypes == 'object')
 object_cols = list(s[s].index )
def getTrainTestBatch() :
fold = 0
while fold < batchValidation.shape[0]:
train_batch = pd.concat([Train.iloc[batchValidation[i][0]:batchValidation[i][1]]
for i in range(batchValidation.shape[0])if i != fold])
 test_batch = Train.iloc[batchValidation[fold][0]:batchValidation[fold][1]]
yield train_batch, test_batch, fold
fold += 1
train_batch, test_batch, fold = next(getTrainTestBatch())
<compute_test_metric>
from sklearn.preprocessing import LabelEncoder
def mse_loss(y_true, y_pred):
return K.sqrt(K.mean(( y_true - y_pred)**2))
def e_loss(y_true, y_pred):
return K.sqrt(K.mean(( np.e**y_true - np.e**y_pred)**2)) 
<choose_model_class>
label_encoder1 = LabelEncoder()
label_encoder2 = LabelEncoder()
train_data['Province_State'] = label_encoder1.fit_transform(train_data['Province_State'])
 test_data['Province_State'] = label_encoder1.transform(test_data['Province_State'])
 train_data['Country_Region'] = label_encoder2.fit_transform(train_data['Country_Region'])
 test_data['Country_Region'] = label_encoder2.transform(test_data['Country_Region'])
class linearAndErrorModels(BaseEstimator, RegressorMixin, TransformerMixin):
def getErrorModel(self, epochs=150):
return xgb.XGBRegressor(colsample_bytree=0.4603, gamma=0.0468,
learning_rate=0.05, max_depth=50,
min_child_weight=1.7817, n_estimators=2200,
reg_alpha=0.4640, reg_lambda=0.8571,
subsample=0.5213,
random_state =7, nthread = -1)
 def __init__(self):
self.models = models
def fit(self, X_Train, X_Train_err, y, epochs=150):
baseModel = LinearRegression()
baseModel.fit(X_Train,Y_Train)
 predict_Train = baseModel.predict(X_Train)
 error_Train = Y_Train - predict_Train
X_Train_err['predict'] = predict_Train
self.X_Train_err = X_Train_err
self.errorModel = self.getErrorModel(epochs=epochs)
 self.errorModel.fit(X_Train_err, error_Train)
 predict_Train = self.errorModel.predict(X_Train_err)
 error_Train = error_Train.values.flatten() - predict_Train.flatten()
X_Train_err['predict_2'] = predict_Train
self.errorModel_2 = self.getErrorModel(epochs=epochs)
 self.errorModel_2.fit(X_Train_err, error_Train)
 self.baseModel = baseModel
return self
def predict(self, X_val, X_val_err):
predict = self.baseModel.predict(X_val)
 X_val_err['predict'] = predict
error = self.errorModel.predict(X_val_err)
 X_val_err['predict_2'] = error.flatten()
error_2 = self.errorModel_2.predict(X_val_err)
 predict = predict.flatten() + error.flatten()
return predict<choose_model_class>
Test_id = test_data.ForecastId
models = ['linear']
genarator = getTrainTestBatch()
Val_Report = {key : [] for key in models}
MSEs = np.array([])
 for train_batch, test_batch, fold in genarator:
sampling = train_batch.sample(frac=1)
 X_Train = sampling.drop('SalePrice_log', axis = 1)
 X_Train_err = Train_err.loc[X_Train.index]
Y_Train = sampling[['SalePrice_log']]
X_val = test_batch.drop('SalePrice_log', axis = 1)
 X_val_err = Train_err.loc[X_val.index]
Y_val = test_batch[['SalePrice_log']]
LAEM = linearAndErrorModels()
LAEM.fit(X_Train, X_Train_err, Y_Train, epochs = 1000)
 predict = LAEM.predict(X_val, X_val_err)
 MSE = mse_loss(K.constant(Y_val), K.constant(predict)).numpy()
MSEs = np.append(MSEs, MSE)
<train_model>
train_data.drop(['Id'], axis=1, inplace=True)
 test_data.drop('ForecastId', axis=1, inplace=True )
X_Train = Train.drop('SalePrice_log', axis = 1)
 X_Train_err = Train_err.loc[X_Train.index]
Y_Train = Train[['SalePrice_log']]
LAEM = linearAndErrorModels()
LAEM.fit(X_Train, X_Train_err, Y_Train )<predict_on_test>
missing_val_count_by_column =(train_data.isnull().sum())
 print(missing_val_count_by_column[missing_val_count_by_column>0] )
predict = LAEM.predict(Test, Test_err )<prepare_output>
from xgboost import XGBRegressor
predict = pd.DataFrame(predict)
 predict.index = Test.index
predict.columns = ['SalePrice_log']
predict['SalePrice'] = np.e**(predict['SalePrice_log'])
 predict = predict.drop('SalePrice_log', axis = 1)
 predict<save_to_csv>
X_train = train_data[['Province_State','Country_Region','Date']]
y_train = train_data[['ConfirmedCases', 'Fatalities']]
predict.to_csv("submission.csv" )<train_model>
y_train_confirm = y_train.ConfirmedCases
y_train_fatality = y_train.Fatalities
numeric_dtypes = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
numeric = []
for i in all_features.columns:
if all_features[i].dtype in numeric_dtypes:
numeric.append(i)
 skew_features = all_features[numeric].apply(lambda x: skew(x)).sort_values(ascending=False)
 high_skew = skew_features[skew_features > 0.5]
skew_index = high_skew.index
print("There are {} numerical features with Skew > 0.5 :".format(high_skew.shape[0]))
skewness = pd.DataFrame({'Skew' :high_skew})
 skew_features.head(5)
 for i in skew_index:
all_features[i] = boxcox1p(all_features[i], boxcox_normmax(all_features[i] + 1))
print("temporal_features:",temporal_features)
 def create_temporal_features(features):
df = features[temporal_features].copy()
for feature in temporal_features:
if feature == 'YrSold' or feature == 'MoSold':
pass
else:
df[feature + "_diff"] = df['YrSold'] - df[feature]
df[feature + "_diff"] = df[feature + "_diff"].fillna(df[feature + "_diff"].max())
 df['YrBltAndRemod'] = df['YearBuilt']+df['YearRemodAdd']
for feature in temporal_features:
df[feature] = df[feature].astype(str)
 df[feature] = df[feature].fillna("Missing")
 df = df.drop(overfit_reducer(df), axis=1)
 return df
def create_categorical_features(features):
df = features[categorical_features].copy()
df['Functional'] = df['Functional'].fillna('Typ')
 df['Utilities'] = df['Utilities'].fillna('AllPub')
 df['Electrical'] = df['Electrical'].fillna("SBrkr")
 df['KitchenQual'] = df['KitchenQual'].fillna("TA")
 df['Exterior1st'] = df['Exterior1st'].fillna(df['Exterior1st'].mode() [0])
 df['Exterior2nd'] = df['Exterior2nd'].fillna(df['Exterior2nd'].mode() [0])
 df['SaleType'] = df['SaleType'].fillna(df['SaleType'].mode() [0])
 for col in ['GarageType', 'GarageFinish', 'GarageQual', 'GarageCond']:
df[col] = df[col].fillna('None')
 for col in('BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2'):
features[col] = features[col].fillna('None')
 features['MSZoning'] = features.groupby('MSSubClass')['MSZoning'].transform(lambda x: x.fillna(x.mode() [0]))
for feature in categorical_features:
df[feature] = df[feature].fillna("Missing")
 df = pd.get_dummies(df)
 df = df.drop(overfit_reducer(df), axis=1)
 return df
def create_numeric_features(features):
df = features[numeric_features + ["Neighborhood"]].copy()
for col in('GarageArea', 'GarageCars'):
df[col] = df[col].fillna(0)
 for feature in numeric_features:
df[feature] = df.groupby("Neighborhood")[feature].transform(lambda x: x.fillna(x.median()))
df = df.drop("Neighborhood", axis=1)
 df['TotalHouseSF'] = df['TotalBsmtSF'] + df['1stFlrSF'] + df['2ndFlrSF']
df['TotalArea'] = df['TotalBsmtSF'] + df['1stFlrSF'] + df['2ndFlrSF'] + df['GrLivArea'] + df['GarageArea']
df['Total_Bathrooms'] =(df['FullBath'] +(0.5 * df['HalfBath'])+
df['BsmtFullBath'] +(0.5 * df['BsmtHalfBath']))
df['TotalBath'] = df['FullBath'] + 0.5 * df['HalfBath']
df['TotalLot'] = df['LotFrontage'] + df['LotArea']
df['TotalPorch'] = df['OpenPorchSF'] + df['EnclosedPorch'] + df['ScreenPorch']
df['Total_sqr_footage'] =(df['BsmtFinSF1'] + df['BsmtFinSF2'] +
df['1stFlrSF'] + df['2ndFlrSF'])
 df['TotalBsmtFin'] = df['BsmtFinSF1'] + df['BsmtFinSF2']
df['Total_porch_sf'] =(df['OpenPorchSF'] + df['3SsnPorch'] +
df['EnclosedPorch'] + df['ScreenPorch'] +
df['WoodDeckSF'])
 df['haspool'] = df['PoolArea'].apply(lambda x: 1 if x > 0 else 0)
 df['has2ndfloor'] = df['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)
 df['hasgarage'] = df['GarageArea'].apply(lambda x: 1 if x > 0 else 0)
 df['hasbsmt'] = df['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 df['hasfireplace'] = df['Fireplaces'].apply(lambda x: 1 if x > 0 else 0)
 df = df.drop(overfit_reducer(df), axis=1)
 df = fixing_skewness(df)
 return df
all_features['BsmtFinType1_Unf'] = 1*(all_features['BsmtFinType1'] == 'Unf')
 all_features['HasWoodDeck'] =(all_features['WoodDeckSF'] == 0)* 1
all_features['HasOpenPorch'] =(all_features['OpenPorchSF'] == 0)* 1
all_features['HasEnclosedPorch'] =(all_features['EnclosedPorch'] == 0)* 1
all_features['Has3SsnPorch'] =(all_features['3SsnPorch'] == 0)* 1
all_features['HasScreenPorch'] =(all_features['ScreenPorch'] == 0)* 1
all_features['Total_Home_Quality'] = all_features['OverallQual'] + all_features['OverallCond']
all_features = all_features.drop(['Utilities', 'Street', 'PoolQC',], axis=1)
 all_features['TotalSF'] = all_features['TotalBsmtSF'] + all_features['1stFlrSF'] + all_features['2ndFlrSF']
all_features['Total_sqr_footage'] =(all_features['BsmtFinSF1'] + all_features['BsmtFinSF2'] +
all_features['1stFlrSF'] + all_features['2ndFlrSF'])
 all_features['Total_Bathrooms'] =(all_features['FullBath'] +(0.5 * all_features['HalfBath'])+
all_features['BsmtFullBath'] +(0.5 * all_features['BsmtHalfBath']))
all_features['Total_porch_sf'] =(all_features['OpenPorchSF'] + all_features['3SsnPorch'] +
all_features['EnclosedPorch'] + all_features['ScreenPorch'] +
all_features['WoodDeckSF'])
 all_features['TotalBsmtSF'] = all_features['TotalBsmtSF'].apply(lambda x: np.exp(6)if x <= 0.0 else x)
 all_features['2ndFlrSF'] = all_features['2ndFlrSF'].apply(lambda x: np.exp(6.5)if x <= 0.0 else x)
 all_features['GarageArea'] = all_features['GarageArea'].apply(lambda x: np.exp(6)if x <= 0.0 else x)
 all_features['GarageCars'] = all_features['GarageCars'].apply(lambda x: 0 if x <= 0.0 else x)
 all_features['LotFrontage'] = all_features['LotFrontage'].apply(lambda x: np.exp(4.2)if x <= 0.0 else x)
 all_features['MasVnrArea'] = all_features['MasVnrArea'].apply(lambda x: np.exp(4)if x <= 0.0 else x)
 all_features['BsmtFinSF1'] = all_features['BsmtFinSF1'].apply(lambda x: np.exp(6.5)if x <= 0.0 else x)
 all_features['haspool'] = all_features['PoolArea'].apply(lambda x: 1 if x > 0 else 0)
 all_features['has2ndfloor'] = all_features['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)
 all_features['hasgarage'] = all_features['GarageArea'].apply(lambda x: 1 if x > 0 else 0)
 all_features['hasbsmt'] = all_features['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 all_features['hasfireplace'] = all_features['Fireplaces'].apply(lambda x: 1 if x > 0 else 0)
 temporal_df = create_temporal_features(all_features[temporal_features])
 all_features = all_features.drop(['YearBuilt', 'YearRemodAdd', 'GarageYrBlt', 'MoSold', 'YrSold'], axis=1)
 all_features = pd.concat([temporal_df,all_features],axis=1)
 all_features['YearRemodAdd'] = all_features['YearRemodAdd'].astype(int)
 all_features = all_features.drop(overfit_reducer(all_features), axis=1)
 def logs(res, ls):
m = res.shape[1]
for l in ls:
res = res.assign(newcol=pd.Series(np.log(1.01+res[l])).values)
 res.columns.values[m] = l + '_log'
m += 1
return res
log_features = ['LotFrontage','LotArea','MasVnrArea','BsmtFinSF1','BsmtFinSF2','BsmtUnfSF',
'TotalBsmtSF','1stFlrSF','2ndFlrSF','LowQualFinSF','GrLivArea',
'BsmtFullBath','BsmtHalfBath','FullBath','HalfBath','BedroomAbvGr','KitchenAbvGr',
'TotRmsAbvGrd','Fireplaces','GarageCars','GarageArea','WoodDeckSF','OpenPorchSF',
'EnclosedPorch','3SsnPorch','ScreenPorch','MiscVal','YearRemodAdd','TotalSF']
all_features = logs(all_features, log_features)
 def squares(res, ls):
m = res.shape[1]
for l in ls:
res = res.assign(newcol=pd.Series(res[l]*res[l] ).values)
 res.columns.values[m] = l + '_sq'
m += 1
return res
squared_features = [ 'LotFrontage_log',
'TotalBsmtSF_log', '1stFlrSF_log', '2ndFlrSF_log', 'GrLivArea_log',
'GarageCars_log', 'GarageArea_log']
all_features = squares(all_features, squared_features)
 all_features = pd.get_dummies(all_features ).reset_index(drop=True)
 all_features = all_features.loc[:,~all_features.columns.duplicated() ] 
<compute_test_metric>
x_train = X_train.iloc[:,:].values
x_test = X_train.iloc[:,:].values
X = all_features.iloc[:len(train_labels), :]
X_test = all_features.iloc[len(train_labels):, :]
kf = KFold(n_splits=12, random_state=42, shuffle=True)
 def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))
def cv_rmse(model, X=X):
rmse = np.sqrt(-cross_val_score(model, X, train_labels, scoring="neg_mean_squared_error", cv=kf))
return(rmse)
<choose_model_class>
model1 = XGBRegressor(n_estimators=400000)
 model1.fit(X_train, y_train_confirm)
 y_pred_confirm = model1.predict(test_data )
lightgbm = LGBMRegressor(objective='regression',
num_leaves=6,
learning_rate=0.01,
n_estimators=7000,
max_bin=200,
bagging_fraction=0.8,
bagging_freq=4,
bagging_seed=8,
feature_fraction=0.2,
feature_fraction_seed=8,
min_sum_hessian_in_leaf = 11,
verbose=-1,
random_state=42)
 xgboost = XGBRegressor(learning_rate=0.01,
n_estimators=6000,
max_depth=4,
min_child_weight=0,
gamma=0.6,
subsample=0.7,
colsample_bytree=0.7,
objective='reg:linear',
nthread=-1,
scale_pos_weight=1,
seed=27,
reg_alpha=0.00006,
random_state=42)
 catboost = CatBoostRegressor(learning_rate= 0.01,
iterations= 6000,
eval_metric= "RMSE",
random_seed= 42,
logging_level= "Silent",
loss_function= "RMSE",
od_type= "Iter",
od_wait= 100,
one_hot_max_size= 20,
l2_leaf_reg= 1.4264966279202298,
depth= 5,
rsm= 0.6253447219486281,
random_strength= 0.009926214908500652,
bagging_temperature= 232)
 ridge_alphas = [1e-15, 1e-10, 1e-8, 9e-4, 7e-4, 5e-4, 3e-4, 1e-4, 1e-3, 5e-2, 1e-2, 0.1, 0.3, 1, 3, 5, 10, 15, 18, 20, 30, 50, 75, 100]
ridge = make_pipeline(RobustScaler() , RidgeCV(alphas=ridge_alphas, cv=kf))
lasso_alphas = [ 5e-5, 1e-4, 5e-4, 1e-3 ]
lasso = make_pipeline(RobustScaler() , LassoCV(max_iter=1e7, alphas=lasso_alphas, cv=kf, random_state=42))
elasticnet_alphas = [5e-5, 1e-4, 5e-4, 1e-3]
elasticnet_l1ratios = [0.8, 0.85, 0.9, 0.95, 1 ]
elasticnet = make_pipeline(RobustScaler() , ElasticNetCV(max_iter=1e7, alphas=elasticnet_alphas, l1_ratio=elasticnet_l1ratios, cv=kf))
kernelridge_alphas = [ ]
kernelridge = make_pipeline(RobustScaler() , KernelRidge(alpha=9.2304694336833,
kernel="polynomial", degree=3, coef0=7.405346770365897))
svr = make_pipeline(RobustScaler() , SVR(C= 20, epsilon= 0.008, gamma=0.0003))
gbr = GradientBoostingRegressor(n_estimators=6000,
learning_rate=0.01,
max_depth=4,
max_features='sqrt',
min_samples_leaf=15,
min_samples_split=10,
loss='huber',
random_state=42)
 rf = RandomForestRegressor(n_estimators=1200,
max_depth=15,
min_samples_split=5,
min_samples_leaf=5,
max_features=None,
oob_score=True,
random_state=42)
 stack_gen = StackingCVRegressor(regressors=(xgboost, lightgbm, catboost, svr, ridge, lasso, elasticnet, kernelridge, gbr, rf),
meta_regressor=xgboost,
use_features_in_secondary=True )<compute_train_metric>
model2 = XGBRegressor(n_estimators=200000)
 model2.fit(X_train,y_train_fatality)
 y_pred_fat = model2.predict(test_data )
scores = {}
score = cv_rmse(lightgbm)
 print("lightgbm: {:.4f}({:.4f})".format(score.mean() , score.std()))
scores['lgb'] =(score.mean() , score.std())
 score = cv_rmse(xgboost)
 print("xgboost: {:.4f}({:.4f})".format(score.mean() , score.std()))
scores['xgb'] =(score.mean() , score.std())
 score = cv_rmse(catboost)
 print("Catboost: {:.4f}({:.4f})".format(score.mean() , score.std()))
scores['Cat'] =(score.mean() , score.std())
 score = cv_rmse(svr)
 print("SVR: {:.4f}({:.4f})".format(score.mean() , score.std()))
scores['svr'] =(score.mean() , score.std())
 score = cv_rmse(ridge)
 print("ridge: {:.4f}({:.4f})".format(score.mean() , score.std()))
scores['ridge'] =(score.mean() , score.std())
 score = cv_rmse(lasso)
 print("lasso: {:.4f}({:.4f})".format(score.mean() , score.std()))
scores['lasso'] =(score.mean() , score.std())
 score = cv_rmse(elasticnet)
 print("elasticnet: {:.4f}({:.4f})".format(score.mean() , score.std()))
scores['elasticnet'] =(score.mean() , score.std())
 score = cv_rmse(kernelridge)
 print("kernelridge: {:.4f}({:.4f})".format(score.mean() , score.std()))
scores['kernelridge'] =(score.mean() , score.std())
 score = cv_rmse(gbr)
 print("gbr: {:.4f}({:.4f})".format(score.mean() , score.std()))
scores['gbr'] =(score.mean() , score.std())
 score = cv_rmse(rf)
 print("rf: {:.4f}({:.4f})".format(score.mean() , score.std()))
scores['rf'] =(score.mean() , score.std())
<train_model>
df_sub = pd.DataFrame()
df_sub['ForecastId'] = Test_id
df_sub['ConfirmedCases'] = y_pred_confirm
df_sub['Fatalities'] = y_pred_fat
df_sub.to_csv('submission.csv', index=False )