Kaggle competition: Mercari Price Suggestion Challenge
<SOS> metric: RMSLE Kaggle data source: mercari-price-suggestion-challenge<init_hyperparams>
%matplotlib inline
sns.set()
plt.rcParams["figure.figsize"] =(20, 10)
 pd.options.display.float_format = '{:.2f}'.format
xgb_params = {
'objective': 'reg:linear',
'booster': 'gbtree',
'learning_rate': 0.02,
'max_depth': 22,
'min_child_weight': 57,
'gamma' : 1.45,
'alpha': 0.0,
'lambda': 0.0,
'subsample': 0.67,
'colsample_bytree': 0.054,
'colsample_bylevel': 0.50,
'n_jobs': -1,
'random_state': 456
}
folds = get_folds(df=full_data[['totals.pageviews']].reset_index() , n_splits=5)
 excluded_features = ['date', 'fullVisitorId', 'sessionId', 'totals.transactionRevenue', 'visitId', 'visitStartTime']
for x in range(40,300,1):
excluded_features.append('pred_'+str(x))
full_features = [_f for _f in full_data.columns if _f not in excluded_features]
print(full_features)
 oof_preds = np.zeros(full_data.shape[0])
 oof_preds1 = np.zeros(full_data.shape[0])
 both_oof = np.zeros(full_data.shape[0])
 sub_preds = np.zeros(sub_full_data.shape[0])
 lgb_temp = np.zeros(sub_full_data.shape[0])
 xgb_temp = np.zeros(sub_full_data.shape[0])
 feature_importance_df = pd.DataFrame()
for fold_,(trn_, val_)in enumerate(folds):
print("-"* 20 + "Fold :"+str(fold_)+ "-"* 20)
 trn_x, trn_y = full_data[full_features].iloc[trn_], trn_user_target['target'].iloc[trn_]
val_x, val_y = full_data[full_features].iloc[val_], trn_user_target['target'].iloc[val_]
xg = XGBRegressor(**xgb_params, n_estimators=20000)
 reg = lgb.LGBMRegressor(
 num_leaves=40,
learning_rate=0.02,
n_estimators=20000,
subsample=.9,
colsample_bytree=.9,
random_state=1 
)
 print("-"* 20 + "LightGBM Training" + "-"* 20)
 reg.fit(
 trn_x, np.log1p(trn_y),
eval_set=[(trn_x, np.log1p(trn_y)) ,(val_x, np.log1p(val_y)) ],
eval_names=['TRAIN', 'VALID'],
early_stopping_rounds=50,
eval_metric='rmse',
verbose=100 
)
 print("-"* 20 + "Xgboost Training" + "-"* 20)
 xg.fit(
 trn_x, np.log1p(trn_y),
eval_set=[(trn_x, np.log1p(trn_y)) ,(val_x, np.log1p(val_y)) ],
early_stopping_rounds=50,
eval_metric='rmse',
verbose=100 
)
 fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = trn_x.columns
fold_importance_df["importance"] = reg.feature_importances_
fold_importance_df["fold"] = fold_ + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 oof_preds[val_] = reg.predict(val_x, num_iteration=reg.best_iteration_)
 oof_preds1[val_] = xg.predict(val_x)
 oof_preds[oof_preds < 0] = 0
oof_preds1[oof_preds1 < 0] = 0
both_oof[val_] = oof_preds[val_] * 0.6 + oof_preds1[val_] * 0.4
_preds = reg.predict(sub_full_data[full_features], num_iteration=reg.best_iteration_)
 _preds[_preds < 0] = 0
pre = xg.predict(sub_full_data[full_features])
 pre[pre<0]=0
sub_preds +=(_preds / len(folds)) * 0.6 +(pre / len(folds)) * 0.4
lgb_temp +=(_preds / len(folds))
xgb_temp +=(pre / len(folds))
def display_importances(feature_importance_df_,output_name):
cols = feature_importance_df_[["feature", "importance"]].groupby("feature", as_index=False ).mean().sort_values(by="importance", ascending=False)[:40]
best_features = feature_importance_df_.loc[feature_importance_df_.feature.isin(cols)]
plt.figure(figsize=(8, 10))
sns.barplot(x="importance", y="feature", data=cols.sort_values(by="importance", ascending=False))
plt.title('LightGBM Features_' +output_name + '_(avg over folds)')
 plt.tight_layout()
plt.savefig('importance_'+ output_name + '.png')
 plt
display_importances(feature_importance_df,"2nd train" )<save_to_csv>
df = pd.read_csv('.. /input/train.tsv', sep='\t', index_col=0 )
print("LGB ", mean_squared_error(np.log1p(trn_user_target['target']), oof_preds)**.5)
 print("XGB ", mean_squared_error(np.log1p(trn_user_target['target']), oof_preds1)**.5)
 print("Combine ", mean_squared_error(np.log1p(trn_user_target['target']), both_oof)**.5)
 sub_full_data['PredictedLogRevenue'] = sub_preds
sub_full_data[['PredictedLogRevenue']].to_csv('futureisthere_gogogo3_both.csv', index=True)
 sub_full_data['PredictedLogRevenue'] = lgb_temp
sub_full_data[['PredictedLogRevenue']].to_csv('futureisthere_gogogo3_lgb.csv', index=True)
 sub_full_data['PredictedLogRevenue'] = xgb_temp
sub_full_data[['PredictedLogRevenue']].to_csv('futureisthere_gogogo3_xgb.csv', index=True )<load_from_csv>
df_test = pd.read_csv('.. /input/test.tsv', sep='\t', index_col=0 )
futureishere_result = pd.read_csv('.. /input/future-is-here/new_test.csv',
dtype={'fullVisitorId': str}, nrows=None )<prepare_output>
sum(df.name.isnull() )
sub_full_data = sub_full_data.reset_index()
sub_full_data['PredictedLogRevenue_lgb+xgb'] = sub_preds
sub_full_data['PredictedLogRevenue_lgb'] = lgb_temp
sub_full_data['PredictedLogRevenue_xgb'] = xgb_temp
sub_full_data['PredictedLogRevenue_futureishere'] = futureishere_result['PredictedLogRevenue']
sub1 = sub_full_data[["fullVisitorId", "PredictedLogRevenue_lgb+xgb"]]
sub2 = sub_full_data[["fullVisitorId", "PredictedLogRevenue_lgb"]]
sub3 = sub_full_data[["fullVisitorId", "PredictedLogRevenue_xgb"]]
sub4 = sub_full_data[["fullVisitorId", "PredictedLogRevenue_futureishere"]]
df_base = pd.merge(sub1,sub2,how='inner',on='fullVisitorId')
 df_base = pd.merge(df_base,sub3,how='inner',on='fullVisitorId')
 df_base = pd.merge(df_base,sub4,how='inner',on='fullVisitorId' )<save_to_csv>
df.category_name.unique().shape
sub_df = pd.DataFrame({"fullVisitorId":sub1['fullVisitorId']})
 sub_df["PredictedLogRevenue"] = df_base['PredictedLogRevenue_futureishere'] * 0.6 + df_base['PredictedLogRevenue_lgb+xgb'] * 0.4
sub_df.columns = ["fullVisitorId", "PredictedLogRevenue"]
sub_df.to_csv('blend3itall.csv', index=False )<set_options>
sum(df.category_name.isnull() )
%matplotlib inline<define_variables>
df.groupby("category_name" ).agg({"price": "mean"} ).sort_values("price", ascending=False ).head(10 )
np.random.seed(127 )<load_from_csv>
sum(df.brand_name.isnull() )
def load_data(filename, nrows=None, raw=False):
if raw == True:
json_columns = ["device", "geoNetwork", "totals", "trafficSource"]
df = pd.read_csv(".. /input/gstore-homework/" + filename,
converters={column: json.loads for column in json_columns},
nrows=nrows)
 else:
df = pd.read_csv(".. /input/gstore-homework/" + filename, nrows=nrows, index_col=0,
dtype={'fullVisitorId': 'str'}, parse_dates=["date"])
 return df<load_from_csv>
df.brand_name.unique().shape
X_train = load_data(filename="train_full_clear_corrected.csv", nrows=None, raw=False)
 y_train = copy.deepcopy(X_train["transactionRevenue"].values)
 y_train_user = X_train[['fullVisitorId', 'transactionRevenue']].groupby('fullVisitorId' ).sum()
X_train.drop(columns=["transactionRevenue"], inplace=True)
 X_test = load_data(filename="test_full_clear_corrected.csv", nrows=None, raw=False)
 bounces_train = pd.read_csv(".. /input/gstore-homework/bounces_train.csv")
 bounces_test = pd.read_csv(".. /input/gstore-homework/bounces_test.csv")
 tmp = pd.DataFrame(X_test["fullVisitorId"] ).merge(bounces_test, left_index=True, right_index=True)
 bounced_users_mask =(tmp[["fullVisitorId", "bounces"]].groupby("fullVisitorId" ).bounces.value_counts().unstack(fill_value=0 ).loc[:, 0] == 0)
 bounced_users_idxs = tmp.set_index("fullVisitorId")[bounced_users_mask].reset_index() ["fullVisitorId"]<define_variables>
df.groupby("brand_name" ).agg({"price": "mean"} ).sort_values("price", ascending=False ).head(10 )
cat_cols = ["channelGrouping", "browser",
"deviceCategory", "operatingSystem",
"city", "continent",
"country", "metro",
"networkDomain", "region",
"subContinent", "campaign", "medium", "isMobile", "source"]
cat_cols_without_time = ["channelGrouping", "browser",
"deviceCategory", "operatingSystem",
"city", "continent",
"country", "metro",
"networkDomain", "region",
"subContinent", "campaign", "medium", "isMobile", "source"]
num_cols = ["hits", "pageviews", "visitNumber", "newVisits"]
num_cols_without_time = ["hits", "pageviews", "visitNumber", "newVisits"]
train_features = cat_cols + num_cols<data_type_conversions>
sum(df.price.isnull() )
for col in cat_cols:
print(col)
 lbl = sklearn.preprocessing.LabelEncoder()
lbl.fit(list(X_train[col].values.astype('str')) + list(X_test[col].values.astype('str')))
 X_train[col] = lbl.transform(list(X_train[col].values.astype('str')))
 X_test[col] = lbl.transform(list(X_test[col].values.astype('str')))
 for col in num_cols:
X_train[col] = X_train[col].astype(float)
 X_test[col] = X_test[col].astype(float )<init_hyperparams>
sum(df.price == 0 )
params = {
"objective" : "regression",
"metric" : "rmse",
"num_leaves" : 20,
"min_child_samples" : 100,
"learning_rate" : 0.1,
"bagging_fraction" : 0.7,
"feature_fraction" : 0.5,
"bagging_frequency" : 5,
"bagging_seed" : 127
}
xgb_params = {
'objective': 'reg:linear',
'booster': 'gbtree',
'learning_rate': 0.1,
'max_depth': 22,
'min_child_weight': 57,
'gamma' : 1.45,
'alpha': 0.0,
'lambda': 0.0,
'subsample': 0.67,
'colsample_bytree': 0.054,
'colsample_bylevel': 0.50,
'n_jobs': -1,
'random_state': 456
}
cat_params = {
'learning_rate' :0.1,
'depth' :10,
'eval_metric' :'RMSE',
'od_type' :'Iter',
'od_wait' : 20,
'random_seed' : 42
}
forest_params = {
"n_estimators": 100,
"min_samples_split": 100,
"min_samples_leaf": 100,
"n_jobs": -1,
"verbose": 1
}<define_variables>
df.groupby("shipping" ).agg({"price": "mean"} )
folds = get_folds(df=X_train, n_splits=5)
 importances = pd.DataFrame()
oof_lgb_preds = np.zeros(X_train.shape[0])
 oof_xgb_preds = np.zeros(X_train.shape[0])
 oof_cat_preds = np.zeros(X_train.shape[0])
 oof_forest_preds = np.zeros(X_train.shape[0])
 lgb_preds = np.zeros(( X_test.shape[0], len(folds)))
 xgb_preds = np.zeros(( X_test.shape[0], len(folds)))
 cat_preds = np.zeros(( X_test.shape[0], len(folds)))
 forest_preds = np.zeros(( X_test.shape[0], len(folds)))
 merge_preds = np.zeros(X_train.shape[0])
 sub_preds = np.zeros(( X_test.shape[0], len(folds)))
 for fold_,(trn_, val_)in enumerate(folds):
print("Fold:",fold_)
 trn_x, trn_y = X_train[train_features].iloc[trn_], y_train[trn_]
val_x, val_y = X_train[train_features].iloc[val_], y_train[val_]
reg = lgb.LGBMRegressor(**params, n_estimators=1100)
 xgb = XGBRegressor(**xgb_params, n_estimators=1000)
 cat = CatBoostRegressor(**cat_params, iterations=1000)
 forest = RandomForestRegressor(**forest_params)
 print("-"* 20 + "LightGBM Training" + "-"* 20)
 reg.fit(trn_x, np.log1p(trn_y),eval_set=[(val_x, np.log1p(val_y)) ],early_stopping_rounds=50,verbose=100,eval_metric='rmse')
 print("-"* 20 + "XGboost Training" + "-"* 20)
 xgb.fit(trn_x, np.log1p(trn_y),eval_set=[(val_x, np.log1p(val_y)) ],early_stopping_rounds=50,eval_metric='rmse',verbose=100)
 print("-"* 20 + "Catboost Training" + "-"* 20)
 cat.fit(trn_x, np.log1p(trn_y), eval_set=[(val_x, np.log1p(val_y)) ],early_stopping_rounds=50,use_best_model=True,verbose=100)
 print("-"* 20 + "Forest Training" + "-"* 20)
 forest.fit(trn_x, np.log1p(trn_y))
imp_df = pd.DataFrame()
imp_df['feature'] = train_features
imp_df['gain_lgb'] = reg.booster_.feature_importance(importance_type='gain')
 imp_df['fold'] = fold_ + 1
importances = pd.concat([importances, imp_df], axis=0, sort=False)
 oof_lgb_preds[val_] = reg.predict(val_x, num_iteration=reg.best_iteration_)
 oof_lgb_preds[oof_lgb_preds < 0] = 0
lgb_preds_cur = reg.predict(X_test[train_features], num_iteration=reg.best_iteration_)
 lgb_preds_cur[lgb_preds_cur < 0] = 0
oof_xgb_preds[val_] = xgb.predict(val_x)
 oof_xgb_preds[oof_xgb_preds < 0] = 0
xgb_preds_cur = xgb.predict(X_test[train_features])
 xgb_preds_cur[xgb_preds_cur < 0] = 0
oof_cat_preds[val_] = cat.predict(val_x)
 oof_cat_preds[oof_cat_preds < 0] = 0
cat_preds_cur = cat.predict(X_test[train_features])
 cat_preds_cur[cat_preds_cur < 0] = 0
oof_forest_preds[val_] = forest.predict(val_x)
 oof_forest_preds[oof_forest_preds < 0] = 0
forest_preds_cur = forest.predict(X_test[train_features])
 forest_preds_cur[forest_preds_cur < 0] = 0
merge_preds[val_] = oof_lgb_preds[val_] * 0.6 + oof_xgb_preds[val_] * 0.2 +\
oof_cat_preds[val_] * 0.1 + oof_forest_preds[val_] * 0.1
lgb_preds[:, fold_] = np.expm1(lgb_preds_cur)
 xgb_preds[:, fold_] = np.expm1(xgb_preds_cur)
 cat_preds[:, fold_] = np.expm1(cat_preds_cur)
 forest_preds[:, fold_] = np.expm1(forest_preds_cur)
 oof_lgb_preds_user = pd.DataFrame(X_train["fullVisitorId"] ).merge(pd.DataFrame(np.expm1(oof_lgb_preds),
columns=["prediction"]), left_index=True, right_index=True)
 oof_lgb_preds_user = oof_lgb_preds_user.groupby("fullVisitorId" ).sum()
comparison = pd.merge(y_train_user, oof_lgb_preds_user,
on=["fullVisitorId"], how="inner")[["transactionRevenue", "prediction"]]
comparison = np.log1p(comparison.values)
 print("Total crossval LGBM rmse: ", mean_squared_error(comparison[:, 0], comparison[:, 1])**.5)
 oof_xgb_preds_user = pd.DataFrame(X_train["fullVisitorId"] ).merge(pd.DataFrame(np.expm1(oof_xgb_preds),
columns=["prediction"]), left_index=True, right_index=True)
 oof_xgb_preds_user = oof_xgb_preds_user.groupby("fullVisitorId" ).sum()
comparison = pd.merge(y_train_user, oof_xgb_preds_user,
on=["fullVisitorId"], how="inner")[["transactionRevenue", "prediction"]]
comparison = np.log1p(comparison.values)
 print("Total crossval XGB rmse: ", mean_squared_error(comparison[:, 0], comparison[:, 1])**.5)
 oof_cat_preds_user = pd.DataFrame(X_train["fullVisitorId"] ).merge(pd.DataFrame(np.expm1(oof_cat_preds),
columns=["prediction"]), left_index=True, right_index=True)
 oof_cat_preds_user = oof_cat_preds_user.groupby("fullVisitorId" ).sum()
comparison = pd.merge(y_train_user, oof_cat_preds_user,
on=["fullVisitorId"], how="inner")[["transactionRevenue", "prediction"]]
comparison = np.log1p(comparison.values)
 print("Total crossval CatBoost rmse: ", mean_squared_error(comparison[:, 0], comparison[:, 1])**.5)
 oof_forest_preds_user = pd.DataFrame(X_train["fullVisitorId"] ).merge(pd.DataFrame(np.expm1(oof_forest_preds),
columns=["prediction"]), left_index=True, right_index=True)
 oof_forest_preds_user = oof_forest_preds_user.groupby("fullVisitorId" ).sum()
comparison = pd.merge(y_train_user, oof_forest_preds_user,
on=["fullVisitorId"], how="inner")[["transactionRevenue", "prediction"]]
comparison = np.log1p(comparison.values)
 print("Total crossval RF rmse: ", mean_squared_error(comparison[:, 0], comparison[:, 1])**.5 )<feature_engineering>
sum(df.item_description.isnull() )
for num in range(len(folds)) :
X_train['predictions_' + "lgb"] = np.expm1(oof_lgb_preds)
 X_train['predictions_' + "xgb"] = np.expm1(oof_xgb_preds)
 X_train['predictions_' + "cat"] = np.expm1(oof_cat_preds)
 X_train['predictions_' + "forest"] = np.expm1(oof_forest_preds)
 X_test['predictions_' + "lgb"] = np.mean(lgb_preds, axis=1)
 X_test['predictions_' + "xgb"] = np.mean(xgb_preds, axis=1)
 X_test['predictions_' + "cat"] = np.mean(cat_preds, axis=1)
 X_test['predictions_' + "forest"] = np.mean(forest_preds, axis=1 )<groupby>
class LabelEncoderPipelineFriendly(LabelEncoder):
def __init__(self, **kwargs):
super(LabelEncoderPipelineFriendly, self ).__init__(**kwargs)
 def fit(self, X, y=None):

 super(LabelEncoderPipelineFriendly, self ).fit(X)
 def transform(self, X, y=None):
return super(LabelEncoderPipelineFriendly, self ).transform(X ).reshape(-1, 1)
 def fit_transform(self, X, y=None):
return super(LabelEncoderPipelineFriendly, self ).fit(X ).transform(X ).reshape(-1, 1 )
trn_data = X_train[num_cols_without_time + ['fullVisitorId']].groupby('fullVisitorId' ).mean()
trn_data[cat_cols_without_time] = X_train[cat_cols_without_time + ['fullVisitorId']].groupby('fullVisitorId' ).median()<groupby>
def prepare_data(train, test):
def get_name_col(df):
return df["name"]
def get_condition_col(df):
return df[["item_condition_id"]]
def get_category_col(df):
return df["category_name"].fillna("None" ).astype("category")
 def get_brand_col(df):
return df["brand_name"].fillna("None" ).astype("category")
 def get_shipping_col(df):
return df[["shipping"]]
def get_desc_col(df):
return df["item_description"].fillna("None")
 p = make_union(*[
make_pipeline(FunctionTransformer(get_name_col, validate=False),
TfidfVectorizer(min_df=15)) ,
make_pipeline(FunctionTransformer(get_condition_col, validate=False),
OneHotEncoder()),
make_pipeline(FunctionTransformer(get_category_col, validate=False),
CountVectorizer()),
make_pipeline(FunctionTransformer(get_brand_col, validate=False),
LabelEncoderPipelineFriendly() ,
OneHotEncoder(sparse=True)) ,
make_pipeline(FunctionTransformer(get_shipping_col, validate=False)) ,
make_pipeline(FunctionTransformer(get_desc_col, validate=False),
TfidfVectorizer(ngram_range=(1, 3),
stop_words="english",
max_features=10000))
])
 train_rows = train.shape[0]
df = pd.concat([train, test], axis=0)
 transformed = p.fit_transform(df)
 transformed_train, transformed_test = transformed[:train_rows], transformed[train_rows:]
del df
return(transformed_train, transformed_test )
full_list = []
for model in ["lgb", "xgb", "cat", "forest"]:
trn_pred_list = X_train[['fullVisitorId', 'predictions_' + model]].groupby('fullVisitorId')\ 
.apply(lambda df: list(df["predictions_" + model])) \ 
.apply(lambda x: {'pred_' + str(i)+ "_" + model: pred for i, pred in enumerate(x)if i <= 43})
 full_list.append(trn_pred_list )<merge>
X, X_test = prepare_data(df, df_test)
 y = np.array(df.price)
 log_y = np.log1p(y )
for num, model in enumerate(["lgb", "xgb", "cat", "forest"]):
if num == 0:
trn_all_predictions = pd.DataFrame(list(full_list[num].values), index=trn_data.index)
 else:
trn_all_predictions = trn_all_predictions.merge(pd.DataFrame(list(full_list[num].values), index=trn_data.index),
left_index=True, right_index=True )<drop_column>
def rmsle(h, y):
return np.sqrt(np.square(np.log(h + 1)- np.log(y + 1)).mean())
 scorer = make_scorer(score_func=rmsle, greater_is_better=False )
del trn_pred_list, full_list<feature_engineering>
models = [ 
("Lasso", Lasso, {"alpha": [0.1, 0.5],
"random_state": [0],
"max_iter": [1000],
"tol": [0.001],
"selection": ["random"],
"fit_intercept": [False]}), 
("Ridge", Ridge, {"solver": ["lsqr", "sparse_cg"],
"alpha": [0.1, 0.5, 1],
"random_state": [0],
"tol": [0.001],
"fit_intercept": [False]}), 
("ElasticNet", ElasticNet, {"alpha": [0.1, 0.5],
"l1_ratio": [0.1, 0.5, 0.9],
"random_state": [0],
"max_iter": [1000],
"tol": [0.001],
"selection": ["random"],
"fit_intercept": [False]}), 
("DecisionTreeRegressor", DecisionTreeRegressor, {"max_depth": [3, 7],
"random_state": [0]})
 ]
for model in ["lgb", "xgb", "cat", "forest"]:
a = X_train[['fullVisitorId', 'predictions_' + model]].groupby('fullVisitorId')
 tmp = copy.deepcopy(X_train[['fullVisitorId', 'predictions_' + model]])
 tmp["predictions_" + model] = np.log1p(tmp["predictions_" + model])
 tmp = tmp.groupby("fullVisitorId")
 trn_data['t_mean_' + model] = np.log1p(a.mean())
 trn_data['t_median_' + model] = np.log1p(a.median())
 trn_data['t_sum_log_' + model] = tmp.sum()
trn_data['t_sum_act_' + model] = np.log1p(a.sum())
 trn_data['t_nb_sess'] = a.count()
full_data = pd.concat([trn_data, trn_all_predictions], axis=1 ).fillna(0)
 train_2level_features = trn_all_predictions.columns<drop_column>
best_models = []
for name, model_class, params in models:
gs = GridSearchCV(model_class() , params, scoring=scorer, cv=5, n_jobs=1, refit=True)
 gs.fit(X, log_y)
 best_models.append(( name, model_class, gs.best_estimator_, gs.best_params_, gs.best_score_))
del trn_data, trn_all_predictions, X_train<groupby>
for model_name, _, estimator, _, _ in best_models:
predicted = np.expm1(estimator.predict(X_test))
pd.DataFrame({"price": predicted}, index=df_test.index ).to_csv("baseline_{}.csv".format(model_name), sep="," )
sub_data = X_test[num_cols_without_time + ['fullVisitorId']].groupby('fullVisitorId' ).mean()
sub_data[cat_cols_without_time] = X_test[cat_cols_without_time + ['fullVisitorId']].groupby('fullVisitorId' ).median()<groupby>
lgr = LGBMRegressor(n_jobs=-1, n_estimators=100)
 lgr.fit(X, log_y, eval_metric=rmsle )
full_list_test = []
for model in ["lgb", "xgb", "cat", "forest"]:
test_pred_list = X_test[['fullVisitorId', 'predictions_' + model]].groupby('fullVisitorId')\ 
.apply(lambda df: list(df["predictions_" + model])) \ 
.apply(lambda x: {'pred_' + str(i)+ "_" + model: pred for i, pred in enumerate(x)if i <= 43})
 full_list_test.append(test_pred_list )<create_dataframe>
best_models.append(( "LGBMRegressor", LGBMRegressor, lgr, lgr.get_params() , lgr.best_score_))
for num, model in enumerate(["lgb", "xgb", "cat", "forest"]):
if num == 0:
test_all_predictions = pd.DataFrame(list(full_list_test[num].values), index=sub_data.index)
 else:
test_all_predictions = test_all_predictions.merge(pd.DataFrame(list(full_list_test[num].values),
index=sub_data.index),
left_index=True, right_index=True)
 for f in train_2level_features:
if f not in test_all_predictions.columns:
test_all_predictions[f] = np.nan<drop_column>
all_models_preds_train = np.zeros(( X.shape[0], len(best_models)))
 for i in range(len(best_models)) :
_, _, estimator, _, _ = best_models[i]
all_models_preds_train[..., i] = estimator.predict(X )
del test_pred_list, full_list_test<feature_engineering>
all_models_preds_test = np.zeros(( X_test.shape[0], len(best_models)))
 for i in range(len(best_models)) :
_, _, estimator, _, _ = best_models[i]
all_models_preds_test[..., i] = estimator.predict(X_test )
for model in ["lgb", "xgb", "cat", "forest"]:
b = X_test[['fullVisitorId', 'predictions_' + model]].groupby('fullVisitorId')
 tmp2 = copy.deepcopy(X_test[['fullVisitorId', 'predictions_' + model]])
 tmp2["predictions_" + model] = np.log1p(tmp2["predictions_" + model])
 tmp2 = tmp2.groupby("fullVisitorId")
 sub_data['t_mean_' + model] = np.log1p(b.mean())
 sub_data['t_median_' + model] = np.log1p(b.median())
 sub_data['t_sum_log_' + model] = tmp2.sum()
sub_data['t_sum_act_' + model] = np.log1p(b.sum())
 sub_data['t_nb_sess'] = b.count()
sub_full_data = pd.concat([sub_data, test_all_predictions], axis=1 ).fillna(0 )<drop_column>
lr = LinearRegression()
lr.fit(all_models_preds_train, log_y)
 final_preds = np.expm1(lr.predict(all_models_preds_test))
<init_hyperparams><EOS> 
pd.DataFrame({"price": final_preds}, index=df_test.index ).to_csv("ensemble.csv", sep="," )