Kaggle competition: COVID19 Global Forecasting (Week 1)
<feature_engineering>
from sklearn.model_selection import train_test_split
import random
from sklearn.impute import SimpleImputer
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import OneHotEncoder
from sklearn.compose import ColumnTransformer
from sklearn.ensemble import RandomForestRegressor
from sklearn.model_selection import GridSearchCV
for f in tqdm_notebook(categorical_features):
train[f], indexer = pd.factorize(train[f])
 test[f] = indexer.get_indexer(test[f] )<prepare_x_and_y>
import datetime
y_reg = train[target_key]<data_type_conversions>
def model() :
rf = RandomForestRegressor(random_state = 42,bootstrap = False, max_depth= 80, max_features = 2,
min_samples_leaf = 5, min_samples_split = 8, n_estimators = 100)
 return rf
train['target'] = y_reg
for df in [train, test]:
df['vis_date'] = pd.to_datetime(df['visitStartTime'], unit='s')
 df['sess_date_dow'] = df['vis_date'].dt.dayofweek
df['sess_date_hours'] = df['vis_date'].dt.hour
df['sess_date_dom'] = df['vis_date'].dt.day
df.sort_values(['fullVisitorId', 'vis_date'], ascending=True, inplace=True)
 df['next_session_1'] =(
 df['vis_date'] - df[['fullVisitorId', 'vis_date']].groupby('fullVisitorId')['vis_date'].shift(1)
 ).astype(np.int64)// 1e9 // 60 // 60
df['next_session_2'] =(
 df['vis_date'] - df[['fullVisitorId', 'vis_date']].groupby('fullVisitorId')['vis_date'].shift(-1)
 ).astype(np.int64)// 1e9 // 60 // 60
df['nb_pageviews'] = df['date'].map(
 df[['date', 'totals.pageviews']].groupby('date')['totals.pageviews'].sum() 
)
 y_reg = train['target']
del train['target']<drop_column>
def train_and_predict(X, y, X_test):
rf_classifier = model()
rf_classifier_model = Pipeline(steps=[ 
('model', rf_classifier)
 ])
 rf_classifier_model.fit(X, y)
 y_pred = rf_classifier_model.predict(X_test)
 y_pred = np.around(y_pred)
 y_pred = y_pred.astype(int)
 return y_pred
del train['vis_date']<prepare_x_and_y>
if __name__ == '__main__':
seed = 123
random.seed(seed)
 print('Loading Training Data')
 covid_train = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-1/train.csv",
parse_dates=['Date'])
 covid_train = covid_train.drop(['Province/State'], axis=1)
 covid_train = covid_train.drop(['Country/Region'], axis=1)
 covid_train['Date'] =(pd.to_datetime(covid_train['Date'], unit='s' ).astype(int)/10**15 ).astype(int)
 cols = [col for col in covid_train.columns if col not in ['Id','ConfirmedCases','Fatalities']]
X = covid_train[cols]
y1 = covid_train['ConfirmedCases']
y2 = covid_train['Fatalities']
print('Loading Testing Data')
 covid_test = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-1/test.csv",
parse_dates=['Date'])
 covid_test = covid_test.drop(['Province/State'], axis=1)
 covid_test = covid_test.drop(['Country/Region'], axis=1)
 covid_test['Date'] =(pd.to_datetime(covid_test['Date'], unit='s' ).astype(int)/10**15 ).astype(int)
 X_test = covid_test.iloc[:,1:]
confirmer_y_pred = train_and_predict(X, y1, X_test)
 fatality_y_pred = train_and_predict(X, y2, X_test)
 pred=pd.DataFrame()
pred['ForecastId']=covid_test['ForecastId']
pred['ConfirmedCases']=confirmer_y_pred
pred['Fatalities']=fatality_y_pred
pred.to_csv('submission.csv',index=False)
train_features = [_f for _f in train.columns if _f not in excluded_features]
importances = pd.DataFrame()
oof_reg_preds = np.zeros(train.shape[0])
 sub_reg_preds = np.zeros(test.shape[0])
 for fold_,(trn_, val_)in enumerate(tqdm_notebook(folds)) :
trn_x, trn_y = train[train_features].iloc[trn_], y_reg.iloc[trn_]
val_x, val_y = train[train_features].iloc[val_], y_reg.iloc[val_]
reg = lgb.LGBMRegressor(
 num_leaves=31,
learning_rate=0.03,
n_estimators=100,
subsample=.9,
colsample_bytree=.9,
random_state=1 
)
 reg.fit(
 trn_x, np.log1p(trn_y),
eval_set=[(val_x, np.log1p(val_y)) ],
early_stopping_rounds=50,
verbose=10 
)
 imp_df = pd.DataFrame()
imp_df['feature'] = train_features
imp_df['gain'] = reg.booster_.feature_importance(importance_type='gain')
 imp_df['fold'] = fold_ + 1
importances = pd.concat([importances, imp_df], axis=0, sort=False)
 oof_reg_preds[val_] = reg.predict(val_x, num_iteration=reg.best_iteration_)
 oof_reg_preds[oof_reg_preds < 0] = 0
_preds = reg.predict(test[train_features], num_iteration=reg.best_iteration_)
 _preds[_preds < 0] = 0
sub_reg_preds += np.expm1(_preds)/ len(folds)
 mean_squared_error(np.log1p(y_reg), oof_reg_preds)**.5<compute_test_metric>
covid_train['Date'] =(pd.to_datetime(covid_train['Date'], unit='s' ).astype(int)/10**9 ).astype(int )
mean_squared_error(np.log1p(y_reg), oof_reg_preds)**.5<prepare_output>
for i in range(len(covid_train['Date'])) :
covid_train['Date'][i] = covid_train['Date'][i].strftime("%d %B, %Y" )
train['predictions'] = np.expm1(oof_reg_preds)
 test['predictions'] = sub_reg_preds
trn_data = train[train_features + ['fullVisitorId']].groupby('fullVisitorId' ).mean()
trn_pred_list = train[['fullVisitorId', 'predictions']].groupby('fullVisitorId')\ 
.apply(lambda df: list(df.predictions)) \ 
.apply(lambda x: {'pred_'+str(i): pred for i, pred in enumerate(x)})
 trn_all_predictions = pd.DataFrame(list(trn_pred_list.values), index=trn_data.index)
 trn_feats = trn_all_predictions.columns
trn_all_predictions['t_mean'] = np.log1p(trn_all_predictions[trn_feats].mean(axis=1))
trn_all_predictions['t_std'] = np.log1p(trn_all_predictions[trn_feats].std(axis=1))
trn_all_predictions['t_median'] = np.log1p(trn_all_predictions[trn_feats].median(axis=1))
trn_all_predictions['t_sum_log'] = np.log1p(trn_all_predictions[trn_feats] ).sum(axis=1)
 trn_all_predictions['t_sum_act'] = np.log1p(trn_all_predictions[trn_feats].fillna(0 ).sum(axis=1))
trn_all_predictions['t_nb_sess'] = trn_all_predictions[trn_feats].isnull().sum(axis=1)
 full_data = pd.concat([trn_data, trn_all_predictions], axis=1)
 del trn_data, trn_all_predictions
gc.collect()<prepare_output>
covid_train['Date'] = str(covid_train['Date'] )
%%time
sub_pred_list = test[['fullVisitorId', 'predictions']].groupby('fullVisitorId')\ 
.apply(lambda df: list(df.predictions)) \ 
.apply(lambda x: {'pred_'+str(i): pred for i, pred in enumerate(x)})
 sub_data = test[train_features + ['fullVisitorId']].groupby('fullVisitorId' ).mean()
del test
sub_all_predictions = pd.DataFrame(list(sub_pred_list.values), index=sub_data.index)
 for f in trn_feats:
if f not in sub_all_predictions.columns:
sub_all_predictions[f] = np.nan
sub_all_predictions['t_mean'] = np.log1p(sub_all_predictions[trn_feats].mean(axis=1))
sub_all_predictions['t_std'] = np.log1p(sub_all_predictions[trn_feats].std(axis=1))
sub_all_predictions['t_median'] = np.log1p(sub_all_predictions[trn_feats].median(axis=1))
sub_all_predictions['t_sum_log'] = np.log1p(sub_all_predictions[trn_feats] ).sum(axis=1)
 sub_all_predictions['t_sum_act'] = np.log1p(sub_all_predictions[trn_feats].fillna(0 ).sum(axis=1))
sub_all_predictions['t_nb_sess'] = sub_all_predictions[trn_feats].isnull().sum(axis=1)
 sub_full_data = pd.concat([sub_data, sub_all_predictions], axis=1)
 del sub_data, sub_all_predictions
gc.collect()
sub_full_data.shape<groupby>
covid_train = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-1/train.csv",
parse_dates=['Date'] )
train['target'] = y_reg
trn_user_target = train[['fullVisitorId', 'target']].groupby('fullVisitorId' ).sum()
del train
gc.collect()<init_hyperparams>
covid_train['Date'].astype(int)/ 10**15