Kaggle competition: Google Analytics Customer Revenue Prediction
raw_train.iloc[[0,-1],[1,-1]]<prepare_x_and_y>
%matplotlib inline
x = raw_train.iloc[:, 1:].values.astype('float32')/ 255
y = raw_train.iloc[:, 0]<split>
np.random.seed(127 )
x_train, x_val, y_train, y_val = train_test_split(x, y, test_size = 0.2, random_state=42 )<categorify>
def load_data(filename, nrows=None, raw=False):
if raw == True:
json_columns = ["device", "geoNetwork", "totals", "trafficSource"]
df = pd.read_csv(".. /input/gstore-homework/" + filename,
converters={column: json.loads for column in json_columns},
nrows=nrows)
 else:
df = pd.read_csv(".. /input/gstore-homework/" + filename, nrows=nrows, index_col=0,
dtype={'fullVisitorId': 'str'}, parse_dates=["date"])
 return df
x_train = x_train.reshape(-1, 28, 28,1)
 x_val = x_val.reshape(-1, 28, 28,1)
 y_train = to_categorical(y_train)
 y_val = to_categorical(y_val )<choose_model_class>
X_train = load_data(filename="train_full_clear_corrected.csv", nrows=None, raw=False)
 y_train = copy.deepcopy(X_train["transactionRevenue"].values)
 y_train_user = X_train[['fullVisitorId', 'transactionRevenue']].groupby('fullVisitorId' ).sum()
X_train.drop(columns=["transactionRevenue"], inplace=True)
 X_test = load_data(filename="test_full_clear_corrected.csv", nrows=None, raw=False)
 bounces_train = pd.read_csv(".. /input/gstore-homework/bounces_train.csv")
 bounces_test = pd.read_csv(".. /input/gstore-homework/bounces_test.csv")
 tmp = pd.DataFrame(X_test["fullVisitorId"] ).merge(bounces_test, left_index=True, right_index=True)
 bounced_users_mask =(tmp[["fullVisitorId", "bounces"]].groupby("fullVisitorId" ).bounces.value_counts().unstack(fill_value=0 ).loc[:, 0] == 0)
 bounced_users_idxs = tmp.set_index("fullVisitorId")[bounced_users_mask].reset_index() ["fullVisitorId"]
model = tf.keras.models.Sequential([
tf.keras.layers.Conv2D(64,(3,3), padding='same', input_shape=(28, 28, 1)) ,
tf.keras.layers.BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer="uniform"),
tf.keras.layers.LeakyReLU(alpha=0.1),
tf.keras.layers.Conv2D(64,(3,3), padding='same'),
tf.keras.layers.BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer="uniform"),
tf.keras.layers.LeakyReLU(alpha=0.1),
tf.keras.layers.Conv2D(64,(3,3), padding='same'),
tf.keras.layers.BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer="uniform"),
tf.keras.layers.LeakyReLU(alpha=0.1),
tf.keras.layers.MaxPooling2D(2, 2),
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Conv2D(128,(3,3), padding='same'),
tf.keras.layers.BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer="uniform"),
tf.keras.layers.LeakyReLU(alpha=0.1),
tf.keras.layers.Conv2D(128,(3,3), padding='same'),
tf.keras.layers.BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer="uniform"),
tf.keras.layers.LeakyReLU(alpha=0.1),
tf.keras.layers.Conv2D(128,(3,3), padding='same'),
tf.keras.layers.BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer="uniform"),
tf.keras.layers.LeakyReLU(alpha=0.1),
tf.keras.layers.MaxPooling2D(2,2),
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Conv2D(256,(3,3), padding='same'),
tf.keras.layers.BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer="uniform"),
tf.keras.layers.LeakyReLU(alpha=0.1),
tf.keras.layers.Conv2D(256,(3,3), padding='same'),
tf.keras.layers.BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer="uniform"),
tf.keras.layers.LeakyReLU(alpha=0.1),
tf.keras.layers.MaxPooling2D(2,2),
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Flatten() ,
tf.keras.layers.Dense(256),
tf.keras.layers.LeakyReLU(alpha=0.1),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dense(10, activation='softmax')
 ] )<choose_model_class>
cat_cols = ["channelGrouping", "browser",
"deviceCategory", "operatingSystem",
"city", "continent",
"country", "metro",
"networkDomain", "region",
"subContinent", "campaign", "medium", "isMobile", "source"]
cat_cols_without_time = ["channelGrouping", "browser",
"deviceCategory", "operatingSystem",
"city", "continent",
"country", "metro",
"networkDomain", "region",
"subContinent", "campaign", "medium", "isMobile", "source"]
num_cols = ["hits", "pageviews", "visitNumber", "newVisits"]
num_cols_without_time = ["hits", "pageviews", "visitNumber", "newVisits"]
train_features = cat_cols + num_cols
optimizer = RMSprop(learning_rate=0.002,
rho=0.9,
momentum=0.1,
epsilon=1e-07,
centered=True,
name='RMSprop')
 model.compile(loss='categorical_crossentropy',
optimizer=optimizer,
metrics=['accuracy'] )<define_variables>
for col in cat_cols:
print(col)
 lbl = sklearn.preprocessing.LabelEncoder()
lbl.fit(list(X_train[col].values.astype('str')) + list(X_test[col].values.astype('str')))
 X_train[col] = lbl.transform(list(X_train[col].values.astype('str')))
 X_test[col] = lbl.transform(list(X_test[col].values.astype('str')))
 for col in num_cols:
X_train[col] = X_train[col].astype(float)
 X_test[col] = X_test[col].astype(float )
batch_size = 1024
num_classes = 10
epochs = 46 
<train_model>
params = {
"objective" : "regression",
"metric" : "rmse",
"num_leaves" : 20,
"min_child_samples" : 100,
"learning_rate" : 0.1,
"bagging_fraction" : 0.7,
"feature_fraction" : 0.5,
"bagging_frequency" : 5,
"bagging_seed" : 127
}
xgb_params = {
'objective': 'reg:linear',
'booster': 'gbtree',
'learning_rate': 0.1,
'max_depth': 22,
'min_child_weight': 57,
'gamma' : 1.45,
'alpha': 0.0,
'lambda': 0.0,
'subsample': 0.67,
'colsample_bytree': 0.054,
'colsample_bylevel': 0.50,
'n_jobs': -1,
'random_state': 456
}
cat_params = {
'learning_rate' :0.1,
'depth' :10,
'eval_metric' :'RMSE',
'od_type' :'Iter',
'od_wait' : 20,
'random_seed' : 42
}
forest_params = {
"n_estimators": 100,
"min_samples_split": 100,
"min_samples_leaf": 100,
"n_jobs": -1,
"verbose": 1
}
datagen_try = ImageDataGenerator(rotation_range=15,
width_shift_range = 0.25,
height_shift_range = 0.25,
shear_range = 25,
zoom_range = 0.4,)
 datagen_try.fit(x_train)
 for x_batch, y_batch in datagen_try.flow(x_train, y_train, batch_size=9):
	
	for i in range(0, 9):
		plt.subplot(330 + 1 + i)
 		plt.imshow(x_batch[i].reshape(28, 28), cmap=plt.get_cmap('gray'))
	
	plt.show()
	break<define_search_model>
folds = get_folds(df=X_train, n_splits=5)
 importances = pd.DataFrame()
oof_lgb_preds = np.zeros(X_train.shape[0])
 oof_xgb_preds = np.zeros(X_train.shape[0])
 oof_cat_preds = np.zeros(X_train.shape[0])
 oof_forest_preds = np.zeros(X_train.shape[0])
 lgb_preds = np.zeros(( X_test.shape[0], len(folds)))
 xgb_preds = np.zeros(( X_test.shape[0], len(folds)))
 cat_preds = np.zeros(( X_test.shape[0], len(folds)))
 forest_preds = np.zeros(( X_test.shape[0], len(folds)))
 merge_preds = np.zeros(X_train.shape[0])
 sub_preds = np.zeros(( X_test.shape[0], len(folds)))
 for fold_,(trn_, val_)in enumerate(folds):
print("Fold:",fold_)
 trn_x, trn_y = X_train[train_features].iloc[trn_], y_train[trn_]
val_x, val_y = X_train[train_features].iloc[val_], y_train[val_]
reg = lgb.LGBMRegressor(**params, n_estimators=1100)
 xgb = XGBRegressor(**xgb_params, n_estimators=1000)
 cat = CatBoostRegressor(**cat_params, iterations=1000)
 forest = RandomForestRegressor(**forest_params)
 print("-"* 20 + "LightGBM Training" + "-"* 20)
 reg.fit(trn_x, np.log1p(trn_y),eval_set=[(val_x, np.log1p(val_y)) ],early_stopping_rounds=50,verbose=100,eval_metric='rmse')
 print("-"* 20 + "XGboost Training" + "-"* 20)
 xgb.fit(trn_x, np.log1p(trn_y),eval_set=[(val_x, np.log1p(val_y)) ],early_stopping_rounds=50,eval_metric='rmse',verbose=100)
 print("-"* 20 + "Catboost Training" + "-"* 20)
 cat.fit(trn_x, np.log1p(trn_y), eval_set=[(val_x, np.log1p(val_y)) ],early_stopping_rounds=50,use_best_model=True,verbose=100)
 print("-"* 20 + "Forest Training" + "-"* 20)
 forest.fit(trn_x, np.log1p(trn_y))
imp_df = pd.DataFrame()
imp_df['feature'] = train_features
imp_df['gain_lgb'] = reg.booster_.feature_importance(importance_type='gain')
 imp_df['fold'] = fold_ + 1
importances = pd.concat([importances, imp_df], axis=0, sort=False)
 oof_lgb_preds[val_] = reg.predict(val_x, num_iteration=reg.best_iteration_)
 oof_lgb_preds[oof_lgb_preds < 0] = 0
lgb_preds_cur = reg.predict(X_test[train_features], num_iteration=reg.best_iteration_)
 lgb_preds_cur[lgb_preds_cur < 0] = 0
oof_xgb_preds[val_] = xgb.predict(val_x)
 oof_xgb_preds[oof_xgb_preds < 0] = 0
xgb_preds_cur = xgb.predict(X_test[train_features])
 xgb_preds_cur[xgb_preds_cur < 0] = 0
oof_cat_preds[val_] = cat.predict(val_x)
 oof_cat_preds[oof_cat_preds < 0] = 0
cat_preds_cur = cat.predict(X_test[train_features])
 cat_preds_cur[cat_preds_cur < 0] = 0
oof_forest_preds[val_] = forest.predict(val_x)
 oof_forest_preds[oof_forest_preds < 0] = 0
forest_preds_cur = forest.predict(X_test[train_features])
 forest_preds_cur[forest_preds_cur < 0] = 0
merge_preds[val_] = oof_lgb_preds[val_] * 0.6 + oof_xgb_preds[val_] * 0.2 +\
oof_cat_preds[val_] * 0.1 + oof_forest_preds[val_] * 0.1
lgb_preds[:, fold_] = np.expm1(lgb_preds_cur)
 xgb_preds[:, fold_] = np.expm1(xgb_preds_cur)
 cat_preds[:, fold_] = np.expm1(cat_preds_cur)
 forest_preds[:, fold_] = np.expm1(forest_preds_cur)
 oof_lgb_preds_user = pd.DataFrame(X_train["fullVisitorId"] ).merge(pd.DataFrame(np.expm1(oof_lgb_preds),
columns=["prediction"]), left_index=True, right_index=True)
 oof_lgb_preds_user = oof_lgb_preds_user.groupby("fullVisitorId" ).sum()
comparison = pd.merge(y_train_user, oof_lgb_preds_user,
on=["fullVisitorId"], how="inner")[["transactionRevenue", "prediction"]]
comparison = np.log1p(comparison.values)
 print("Total crossval LGBM rmse: ", mean_squared_error(comparison[:, 0], comparison[:, 1])**.5)
 oof_xgb_preds_user = pd.DataFrame(X_train["fullVisitorId"] ).merge(pd.DataFrame(np.expm1(oof_xgb_preds),
columns=["prediction"]), left_index=True, right_index=True)
 oof_xgb_preds_user = oof_xgb_preds_user.groupby("fullVisitorId" ).sum()
comparison = pd.merge(y_train_user, oof_xgb_preds_user,
on=["fullVisitorId"], how="inner")[["transactionRevenue", "prediction"]]
comparison = np.log1p(comparison.values)
 print("Total crossval XGB rmse: ", mean_squared_error(comparison[:, 0], comparison[:, 1])**.5)
 oof_cat_preds_user = pd.DataFrame(X_train["fullVisitorId"] ).merge(pd.DataFrame(np.expm1(oof_cat_preds),
columns=["prediction"]), left_index=True, right_index=True)
 oof_cat_preds_user = oof_cat_preds_user.groupby("fullVisitorId" ).sum()
comparison = pd.merge(y_train_user, oof_cat_preds_user,
on=["fullVisitorId"], how="inner")[["transactionRevenue", "prediction"]]
comparison = np.log1p(comparison.values)
 print("Total crossval CatBoost rmse: ", mean_squared_error(comparison[:, 0], comparison[:, 1])**.5)
 oof_forest_preds_user = pd.DataFrame(X_train["fullVisitorId"] ).merge(pd.DataFrame(np.expm1(oof_forest_preds),
columns=["prediction"]), left_index=True, right_index=True)
 oof_forest_preds_user = oof_forest_preds_user.groupby("fullVisitorId" ).sum()
comparison = pd.merge(y_train_user, oof_forest_preds_user,
on=["fullVisitorId"], how="inner")[["transactionRevenue", "prediction"]]
comparison = np.log1p(comparison.values)
 print("Total crossval RF rmse: ", mean_squared_error(comparison[:, 0], comparison[:, 1])**.5 )
datagen_train = ImageDataGenerator(rotation_range = 10,
width_shift_range = 0.25,
height_shift_range = 0.25,
shear_range = 10,
zoom_range = 0.1,
horizontal_flip = False)
 datagen_val = ImageDataGenerator()
step_train = x_train.shape[0] // batch_size
step_val = x_val.shape[0] // batch_size
learning_rate_reduction = tf.keras.callbacks.ReduceLROnPlateau(
 monitor='loss',
factor=0.25,
patience=2,
verbose=1,
mode="auto",
min_delta=0.0001,
cooldown=0,
min_lr=0.00001 
)
 es = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=300, restore_best_weights=True)
<train_model>
for num in range(len(folds)) :
X_train['predictions_' + "lgb"] = np.expm1(oof_lgb_preds)
 X_train['predictions_' + "xgb"] = np.expm1(oof_xgb_preds)
 X_train['predictions_' + "cat"] = np.expm1(oof_cat_preds)
 X_train['predictions_' + "forest"] = np.expm1(oof_forest_preds)
 X_test['predictions_' + "lgb"] = np.mean(lgb_preds, axis=1)
 X_test['predictions_' + "xgb"] = np.mean(xgb_preds, axis=1)
 X_test['predictions_' + "cat"] = np.mean(cat_preds, axis=1)
 X_test['predictions_' + "forest"] = np.mean(forest_preds, axis=1 )
history = model.fit_generator(datagen_train.flow(x_train, y_train, batch_size=batch_size),
steps_per_epoch=len(x_train)//batch_size,
epochs=epochs,
validation_data=(x_val, y_val),
validation_steps=50,
callbacks=[learning_rate_reduction, es],
verbose=2 )<train_model>
trn_data = X_train[num_cols_without_time + ['fullVisitorId']].groupby('fullVisitorId' ).mean()
trn_data[cat_cols_without_time] = X_train[cat_cols_without_time + ['fullVisitorId']].groupby('fullVisitorId' ).median()
tf.keras.utils.plot_model(model, to_file="model.png", show_shapes=True )<compute_test_metric>
full_list = []
for model in ["lgb", "xgb", "cat", "forest"]:
trn_pred_list = X_train[['fullVisitorId', 'predictions_' + model]].groupby('fullVisitorId')\ 
.apply(lambda df: list(df["predictions_" + model])) \ 
.apply(lambda x: {'pred_' + str(i)+ "_" + model: pred for i, pred in enumerate(x)if i <= 43})
 full_list.append(trn_pred_list )
model.evaluate(x_val, y_val, verbose=2);<predict_on_test>
for num, model in enumerate(["lgb", "xgb", "cat", "forest"]):
if num == 0:
trn_all_predictions = pd.DataFrame(list(full_list[num].values), index=trn_data.index)
 else:
trn_all_predictions = trn_all_predictions.merge(pd.DataFrame(list(full_list[num].values), index=trn_data.index),
left_index=True, right_index=True )
y_predicted = model.predict(x_val)
 y_grand_truth = y_val
y_predicted = np.argmax(y_predicted,axis=1)
 y_grand_truth = np.argmax(y_grand_truth,axis=1)
 cm = confusion_matrix(y_grand_truth, y_predicted )<prepare_output>
del trn_pred_list, full_list
scores = np.zeros(( 10,3))
def calc_F1(num):
TP = cm[num,num]
FN = np.sum(cm[num,:])-cm[num,num]
FP = np.sum(cm[:,num])-cm[num,num]
precision = TP/(TP+FP)
 recall = TP/(TP+FN)
 F1_score = 2*(recall * precision)/(recall + precision)
 return precision, recall, F1_score
for i in range(10):
precision, recall, F1_score = calc_F1(i)
 scores[i,:] = precision, recall, F1_score
scores_frame = pd.DataFrame(scores,columns=["Precision", "Recall", "F1 Score"], index=[list(range(0, 10)) ] )<load_from_csv>
for model in ["lgb", "xgb", "cat", "forest"]:
a = X_train[['fullVisitorId', 'predictions_' + model]].groupby('fullVisitorId')
 tmp = copy.deepcopy(X_train[['fullVisitorId', 'predictions_' + model]])
 tmp["predictions_" + model] = np.log1p(tmp["predictions_" + model])
 tmp = tmp.groupby("fullVisitorId")
 trn_data['t_mean_' + model] = np.log1p(a.mean())
 trn_data['t_median_' + model] = np.log1p(a.median())
 trn_data['t_sum_log_' + model] = tmp.sum()
trn_data['t_sum_act_' + model] = np.log1p(a.sum())
 trn_data['t_nb_sess'] = a.count()
full_data = pd.concat([trn_data, trn_all_predictions], axis=1 ).fillna(0)
 train_2level_features = trn_all_predictions.columns
raw_dig = pd.read_csv(".. /input/Kannada-MNIST/Dig-MNIST.csv")
 raw_dig.head()
x_dig = raw_dig.iloc[:, 1:].values.astype('float32')/ 255
y_dig = raw_dig.iloc[:, 0].values
x_dig = x_dig.reshape(-1,28,28,1)
 y_dig = to_categorical(y_dig)
 model.evaluate(x_dig, y_dig, verbose=2 )<load_from_csv>
del trn_data, trn_all_predictions, X_train
sample_sub=pd.read_csv('.. /input/Kannada-MNIST/sample_submission.csv')
 raw_test_id=raw_test.id
raw_test=raw_test.drop("id",axis="columns")
 raw_test=raw_test / 255
test=raw_test.values.reshape(-1,28,28,1)
 test.shape<save_to_csv>
sub_data = X_test[num_cols_without_time + ['fullVisitorId']].groupby('fullVisitorId' ).mean()
sub_data[cat_cols_without_time] = X_test[cat_cols_without_time + ['fullVisitorId']].groupby('fullVisitorId' ).median()
sub=model.predict(test)
 sub=np.argmax(sub,axis=1)
 sample_sub['label']=sub
sample_sub.to_csv('submission.csv',index=False )<set_options>
full_list_test = []
for model in ["lgb", "xgb", "cat", "forest"]:
test_pred_list = X_test[['fullVisitorId', 'predictions_' + model]].groupby('fullVisitorId')\ 
.apply(lambda df: list(df["predictions_" + model])) \ 
.apply(lambda x: {'pred_' + str(i)+ "_" + model: pred for i, pred in enumerate(x)if i <= 43})
 full_list_test.append(test_pred_list )
warnings.filterwarnings('ignore')
 for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))<load_from_csv>
for num, model in enumerate(["lgb", "xgb", "cat", "forest"]):
if num == 0:
test_all_predictions = pd.DataFrame(list(full_list_test[num].values), index=sub_data.index)
 else:
test_all_predictions = test_all_predictions.merge(pd.DataFrame(list(full_list_test[num].values),
index=sub_data.index),
left_index=True, right_index=True)
 for f in train_2level_features:
if f not in test_all_predictions.columns:
test_all_predictions[f] = np.nan
raw_train = pd.read_csv('.. /input/Kannada-MNIST/train.csv')
 raw_test = pd.read_csv('.. /input/Kannada-MNIST/test.csv' )<define_search_space>
del test_pred_list, full_list_test
raw_train.iloc[[0,-1],[1,-1]]<prepare_x_and_y>
for model in ["lgb", "xgb", "cat", "forest"]:
b = X_test[['fullVisitorId', 'predictions_' + model]].groupby('fullVisitorId')
 tmp2 = copy.deepcopy(X_test[['fullVisitorId', 'predictions_' + model]])
 tmp2["predictions_" + model] = np.log1p(tmp2["predictions_" + model])
 tmp2 = tmp2.groupby("fullVisitorId")
 sub_data['t_mean_' + model] = np.log1p(b.mean())
 sub_data['t_median_' + model] = np.log1p(b.median())
 sub_data['t_sum_log_' + model] = tmp2.sum()
sub_data['t_sum_act_' + model] = np.log1p(b.sum())
 sub_data['t_nb_sess'] = b.count()
sub_full_data = pd.concat([sub_data, test_all_predictions], axis=1 ).fillna(0 )
x = raw_train.iloc[:, 1:].values.astype('float32')/ 255
y = raw_train.iloc[:, 0]<split>
del sub_data, test_all_predictions
x_train, x_val, y_train, y_val = train_test_split(x, y, test_size = 0.2, random_state=42 )<categorify>
params = {
"objective" : "regression",
"metric" : "rmse",
"num_leaves" : 20,
"min_child_samples" : 100,
"learning_rate" : 0.01,
"bagging_fraction" : 0.7,
"feature_fraction" : 0.5,
"bagging_frequency" : 5,
"bagging_seed" : 127
}
xgb_params = {
'objective': 'reg:linear',
'booster': 'gbtree',
'learning_rate': 0.01,
'max_depth': 22,
'min_child_weight': 57,
'gamma' : 1.45,
'alpha': 0.0,
'lambda': 0.0,
'subsample': 0.67,
'colsample_bytree': 0.054,
'colsample_bylevel': 0.50,
'n_jobs': -1,
'random_state': 456
}
cat_params = {
'learning_rate' :0.01,
'depth' :10,
'eval_metric' :'RMSE',
'od_type' :'Iter',
'od_wait' : 20,
'random_seed' : 42
}
x_train = x_train.reshape(-1, 28, 28,1)
 x_val = x_val.reshape(-1, 28, 28,1)
 y_train = to_categorical(y_train)
 y_val = to_categorical(y_val )<choose_model_class>
folds = get_folds(df=full_data["pageviews"].reset_index() , n_splits=2)
 oof_reg_preds = np.zeros(full_data.shape[0])
 oof_reg_preds1 = np.zeros(full_data.shape[0])
 oof_reg_preds2 = np.zeros(full_data.shape[0])
 sub_preds = np.zeros(sub_full_data.shape[0])
 vis_importances = pd.DataFrame()
for fold_,(trn_, val_)in enumerate(folds):
print("Fold:",fold_)
 trn_x, trn_y = full_data.iloc[trn_], y_train_user['transactionRevenue'].iloc[trn_]
val_x, val_y = full_data.iloc[val_], y_train_user['transactionRevenue'].iloc[val_]
reg = lgb.LGBMRegressor(**params, n_estimators=1100)
 xgb = XGBRegressor(**xgb_params, n_estimators=1000)
 cat = CatBoostRegressor(**cat_params, iterations=1000)
 print("-"* 20 + "LightGBM Training" + "-"* 20)
 reg.fit(trn_x, np.log1p(trn_y),eval_set=[(val_x, np.log1p(val_y)) ],early_stopping_rounds=50,verbose=100,eval_metric='rmse')
 print("-"* 20 + "XGboost Training" + "-"* 20)
 xgb.fit(trn_x, np.log1p(trn_y),eval_set=[(val_x, np.log1p(val_y)) ],early_stopping_rounds=50,eval_metric='rmse',verbose=100)
 print("-"* 20 + "Catboost Training" + "-"* 20)
 cat.fit(trn_x, np.log1p(trn_y), eval_set=[(val_x, np.log1p(val_y)) ],early_stopping_rounds=50,use_best_model=True,verbose=100)
 imp_df = pd.DataFrame()
imp_df['feature'] = trn_x.columns
imp_df['gain'] = reg.booster_.feature_importance(importance_type='gain')
 imp_df['fold'] = fold_ + 1
vis_importances = pd.concat([vis_importances, imp_df], axis=0, sort=False)
 oof_reg_preds[val_] = reg.predict(val_x, num_iteration=reg.best_iteration_)
 oof_reg_preds[oof_reg_preds < 0] = 0
lgb_preds = reg.predict(sub_full_data, num_iteration=reg.best_iteration_)
 lgb_preds[lgb_preds < 0] = 0
oof_reg_preds1[val_] = xgb.predict(val_x)
 oof_reg_preds1[oof_reg_preds1 < 0] = 0
xgb_preds = xgb.predict(sub_full_data)
 xgb_preds[xgb_preds < 0] = 0
oof_reg_preds2[val_] = cat.predict(val_x)
 oof_reg_preds2[oof_reg_preds2 < 0] = 0
cat_preds = cat.predict(sub_full_data)
 cat_preds[cat_preds < 0] = 0
sub_preds +=(lgb_preds / len(folds)) * 0.6 +(xgb_preds / len(folds)) * 0.3 +(cat_preds / len(folds)) * 0.1
print("LGBM Result ", mean_squared_error(np.log1p(y_train_user['transactionRevenue']), oof_reg_preds)**.5)
 print("XGBoost Result", mean_squared_error(np.log1p(y_train_user['transactionRevenue']), oof_reg_preds1)**.5)
 print("CatBoost Result", mean_squared_error(np.log1p(y_train_user['transactionRevenue']), oof_reg_preds2)**.5 )
model = tf.keras.models.Sequential([
tf.keras.layers.Conv2D(64,(3,3), padding='same', input_shape=(28, 28, 1)) ,
tf.keras.layers.BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer="uniform"),
tf.keras.layers.LeakyReLU(alpha=0.1),
tf.keras.layers.Conv2D(64,(3,3), padding='same'),
tf.keras.layers.BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer="uniform"),
tf.keras.layers.LeakyReLU(alpha=0.1),
tf.keras.layers.Conv2D(64,(3,3), padding='same'),
tf.keras.layers.BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer="uniform"),
tf.keras.layers.LeakyReLU(alpha=0.1),
tf.keras.layers.MaxPooling2D(2, 2),
tf.keras.layers.Conv2D(128,(3,3), padding='same'),
tf.keras.layers.BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer="uniform"),
tf.keras.layers.LeakyReLU(alpha=0.1),
tf.keras.layers.Conv2D(128,(3,3), padding='same'),
tf.keras.layers.BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer="uniform"),
tf.keras.layers.LeakyReLU(alpha=0.1),
tf.keras.layers.Conv2D(128,(3,3), padding='same'),
tf.keras.layers.BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer="uniform"),
tf.keras.layers.LeakyReLU(alpha=0.1),
tf.keras.layers.MaxPooling2D(2,2),
tf.keras.layers.Conv2D(256,(3,3), padding='same'),
tf.keras.layers.BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer="uniform"),
tf.keras.layers.LeakyReLU(alpha=0.1),
tf.keras.layers.Conv2D(256,(3,3), padding='same'),
tf.keras.layers.BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer="uniform"),
tf.keras.layers.LeakyReLU(alpha=0.1),
tf.keras.layers.MaxPooling2D(2,2),
tf.keras.layers.Flatten() ,
tf.keras.layers.Dense(256),
tf.keras.layers.LeakyReLU(alpha=0.1),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dense(10, activation='softmax')
 ] )<choose_model_class>
sub_df = pd.DataFrame({"fullVisitorId": sub_full_data.reset_index() ["fullVisitorId"].values})
 sub_df["PredictedLogRevenue"] = sub_preds
idxs = sub_df["fullVisitorId"].isin(bounced_users_idxs)
 sub_df.loc[idxs, "PredictedLogRevenue"] = 0
<define_variables><EOS> 
sub_df.to_csv('two_levels_ensemble.csv', index=False )