Kaggle competition: Elo Merchant Category Recommendation
print('mae train: ', mean_absolute_error(rf.predict(x_train), y_train))
print('mae val: ', mean_absolute_error(rf.predict(x_val), y_val))<save_to_csv>
train = pd.read_csv(".. /input/elo-merchant-category-recommendation/train.csv")
 test = pd.read_csv(".. /input/elo-merchant-category-recommendation/test.csv")
 train_df = pd.read_csv(".. /input/fork-of-stacked-lgbm-reduced-models/trainconsol.csv")
 test_df = pd.read_csv(".. /input/fork-of-stacked-lgbm-reduced-models/testconsol.csv")
 train_rfm = pd.read_csv(".. /input/customer-loyalty-based-on-rfm-analysis/trainrfm.csv")
 test_rfm = pd.read_csv(".. /input/customer-loyalty-based-on-rfm-analysis/testrfm.csv")
 feat_imp = pd.read_csv(".. /input/feature-importance-selection/scores_df.csv" )
%%time
pred = rf.predict(x_test)
 test['winPlacePerc'] = pred
submission = test[['Id', 'winPlacePerc']]
submission.to_csv('submission.csv', index=False )<import_modules>
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import itertools
import gc
import os
import sys
from sklearn import preprocessing
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error,mean_absolute_error
from sklearn.ensemble import RandomForestRegressor
from timeit import default_timer as timer
import lightgbm as lgb
from keras.models import Sequential
from keras.layers import Dense
from keras.callbacks import ModelCheckpoint<load_from_csv>
train_rfm.drop(columns=['first_active_month', 'feature_1', 'feature_2', 'feature_3',
'target', 'year', 'month', 'weekday', 'frequency',
'value', 'recency', 'frequency_new', 'value_new', 'recency_new',
'reference_date', 'category_month_lag','elapsedtime'],inplace= True)
 test_rfm.drop(columns=['first_active_month', 'feature_1', 'feature_2', 'feature_3',
'year', 'month', 'weekday', 'frequency',
'value', 'recency', 'frequency_new', 'value_new', 'recency_new',
'reference_date', 'category_month_lag','elapsedtime'],inplace= True )
df_train = pd.read_csv('.. /input/pubg-finish-placement-prediction/train_V2.csv')
 df_test = pd.read_csv('.. /input/pubg-finish-placement-prediction/test_V2.csv')
 df_train=reduce_mem_usage(df_train)
 df_test=reduce_mem_usage(df_test)
 df_train.head()
df_train.describe()<drop_column>
train_df= pd.merge(train_df,train_rfm,on='card_id',how='left')
 test_df= pd.merge(test_df,test_rfm,on='card_id',how='left' )
<feature_engineering>
def feature_engineering(df,is_train=True):
if is_train:
df = df[df['maxPlace'] > 1]
state('totalDistance')
 s = timer()
df['totalDistance'] = df['rideDistance'] + df["walkDistance"] + df["swimDistance"]
e = timer()
state('totalDistance', False, e - s)
 state('rankPoints')
 s = timer()
df['rankPoints'] = np.where(df['rankPoints'] <= 0 ,0 , df['rankPoints'])
 e = timer()
state('rankPoints', False, e-s)
 target = 'winPlacePerc'
features = list(df.columns)
 features.remove("Id")
 features.remove("matchId")
 features.remove("groupId")
 features.remove("matchDuration")
 features.remove("matchType")
 y = None
if is_train:
y = np.array(df.groupby(['matchId','groupId'])[target].agg('mean'), dtype=np.float64)
 features.remove(target)
 print("get group mean feature")
 agg = df.groupby(['matchId','groupId'])[features].agg('mean')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
if is_train:
df_out = agg.reset_index() [['matchId','groupId']]
else:
df_out = df[['matchId','groupId']]
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_mean", "_mean_rank"], how='left', on=['matchId', 'groupId'])
 print("get group max feature")
 agg = df.groupby(['matchId','groupId'])[features].agg('max')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_max", "_max_rank"], how='left', on=['matchId', 'groupId'])
 print("get group min feature")
 agg = df.groupby(['matchId','groupId'])[features].agg('min')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_min", "_min_rank"], how='left', on=['matchId', 'groupId'])
 print("get group size feature")
 agg = df.groupby(['matchId','groupId'] ).size().reset_index(name='group_size')
 df_out = df_out.merge(agg, how='left', on=['matchId', 'groupId'])
 print("get match mean feature")
 agg = df.groupby(['matchId'])[features].agg('mean' ).reset_index()
df_out = df_out.merge(agg, suffixes=["", "_match_mean"], how='left', on=['matchId'])
 print("get match size feature")
 agg = df.groupby(['matchId'] ).size().reset_index(name='match_size')
 df_out = df_out.merge(agg, how='left', on=['matchId'])
 df_out.drop(["matchId", "groupId"], axis=1, inplace=True)
 df_out = reduce_mem_usage(df_out)
 X = np.array(df_out, dtype=np.float64)
 del df, df_out, agg, agg_rank
gc.collect()
return X, y 
<prepare_x_and_y>
target = train_df.target
train_df= train_df.drop(['card_id','target'],axis=1)
 card_id = test_df['card_id']
test_df= test_df.drop(['card_id'],axis=1 )
x_train, y_train = feature_engineering(df_train,True)
 x_test,_ = feature_engineering(df_test,False )<split>
features = [c for c in train_df.columns if c not in lessimp_feature ]
categorical_feats = ['feature_2','feature_1','feature_comb_index', 'month','category_month_lag','f_quantile','r_quantile','RFMScore','rnew_quantile','vnew_quantile' ]
random_seed=1
x_train, x_val, y_train, y_val = train_test_split(x_train, y_train, test_size = 0.05, random_state=random_seed )<choose_model_class>
param = {
'objective':'regression',
'learning_rate': 0.01,
'boosting': 'gbdt',
'num_leaves': 31,
'colsample_bytree': 0.5178284844630764,
'subsample': 0.6701600397051216,
'max_depth': 10,
'reg_alpha': 2.5056142297682493,
'reg_lambda': 2.4364948978876253,
'min_split_gain': 4.947802532689812,
'min_child_weight': 43.41313044141597,
'min_data_in_leaf': 40,
"metric": 'rmse',
'bagging_seed':11,
"verbosity": -1
}
param1= {'num_leaves': 50,
'min_data_in_leaf': 30,
'objective':'regression',
'max_depth': 10,
'learning_rate': 0.005,
"min_child_samples": 100,
"boosting": "gbdt",
"metric": 'rmse',
"feature_fraction": 0.9,
"bagging_freq": 1,
"bagging_fraction": 0.9 ,
"bagging_seed": 11,
"lambda_l1": 0.1,
"verbosity": -1
}
RF = RandomForestRegressor(n_estimators=10, min_samples_leaf=3, max_features=0.5, n_jobs=-1 )<train_model>
folds = StratifiedKFold(n_splits=5, shuffle=True, random_state=15)
 oof = np.zeros(len(train_df))
predictions = np.zeros(len(test_df))
start = time.time()
feature_importance_df = pd.DataFrame()
for fold_,(trn_idx, val_idx)in enumerate(folds.split(train_df.values,train_df['feature_comb_index'].values)) :
print("fold n°{}".format(fold_))
trn_data = lgb.Dataset(train_df.iloc[trn_idx][features], label=target.iloc[trn_idx], categorical_feature=categorical_feats)
 val_data = lgb.Dataset(train_df.iloc[val_idx][features], label=target.iloc[val_idx], categorical_feature=categorical_feats)
 num_round = 10000
params= param
clf = lgb.train(params, trn_data, num_round, valid_sets = [trn_data, val_data], verbose_eval=-1, early_stopping_rounds = 200)
 oof[val_idx] = clf.predict(train_df.iloc[val_idx][features], num_iteration=clf.best_iteration)
 fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = features
fold_importance_df["importance"] = clf.feature_importance()
fold_importance_df["fold"] = fold_ + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 predictions += clf.predict(test_df[features], num_iteration=clf.best_iteration)/ folds.n_splits
print("CV score: {:<8.5f}".format(mean_squared_error(oof, target)**0.5))
%%time
RF.fit(x_train, y_train )<compute_train_metric>
submit = pd.DataFrame({'card_id':card_id,'target':predictions})
 submit.to_csv("featurecombLGBM1.csv",index=False )
mae_train_RF = mean_absolute_error(RF.predict(x_train), y_train)
 mae_val_RF = mean_absolute_error(RF.predict(x_val), y_val)
 print('mae train RF: ', mae_train_RF)
 print('mae val RF: ', mae_val_RF )<train_model>
folds = StratifiedKFold(n_splits=5, shuffle=True, random_state=15)
 oof1 = np.zeros(len(train_df))
predictions1 = np.zeros(len(test_df))
start = time.time()
feature_importance_df1 = pd.DataFrame()
for fold_,(trn_idx, val_idx)in enumerate(folds.split(train_df.values,train_df['category_month_lag'].values)) :
print("fold n°{}".format(fold_))
trn_data = lgb.Dataset(train_df.iloc[trn_idx][features], label=target.iloc[trn_idx], categorical_feature=categorical_feats)
 val_data = lgb.Dataset(train_df.iloc[val_idx][features], label=target.iloc[val_idx], categorical_feature=categorical_feats)
 num_round = 10000
params= param
clf1 = lgb.train(params, trn_data, num_round, valid_sets = [trn_data, val_data], verbose_eval=-1, early_stopping_rounds = 200)
 oof1[val_idx] = clf1.predict(train_df.iloc[val_idx][features], num_iteration=clf1.best_iteration)
 fold_importance_df1 = pd.DataFrame()
fold_importance_df1["feature"] = features
fold_importance_df1["importance"] = clf1.feature_importance()
fold_importance_df1["fold"] = fold_ + 1
feature_importance_df1 = pd.concat([feature_importance_df1, fold_importance_df1], axis=0)
 predictions1 += clf1.predict(test_df[features], num_iteration=clf1.best_iteration)/ folds.n_splits
print("CV score: {:<8.5f}".format(mean_squared_error(oof1, target)**0.5))
def run_lgb(train_X, train_y, val_X, val_y, x_test):
params = {"objective" : "regression",
"metric" : "mae",
'n_estimators':20000,
'early_stopping_rounds':200,
"num_leaves" : 31,
"learning_rate" : 0.05,
"bagging_fraction" : 0.7,
"bagging_seed" : 0,
"num_threads" : 4,
"colsample_bytree" : 0.7
}
lgtrain = lgb.Dataset(train_X, label=train_y)
 lgval = lgb.Dataset(val_X, label=val_y)
 model = lgb.train(params, lgtrain, valid_sets=[lgtrain, lgval], early_stopping_rounds=200, verbose_eval=1000)
 pred_test_y = model.predict(x_test, num_iteration=model.best_iteration)
 return pred_test_y, model<train_model>
submissioncatmonth = pd.read_csv('.. /input/elo-merchant-category-recommendation/sample_submission.csv')
 submissioncatmonth['target'] = predictions1
submissioncatmonth.to_csv('LGBMcategorymonthlag1.csv', index=False)
%%time
pred_test_lgb, model = run_lgb(x_train, y_train, x_val, y_val, x_test )<compute_train_metric>
folds = StratifiedKFold(n_splits=5, shuffle=True, random_state=15)
 oof2 = np.zeros(len(train_df))
predictions2 = np.zeros(len(test_df))
start = time.time()
feature_importance_df2 = pd.DataFrame()
for fold_,(trn_idx, val_idx)in enumerate(folds.split(train_df.values,train_df['RFMScore'].values)) :
print("fold n°{}".format(fold_))
trn_data = lgb.Dataset(train_df.iloc[trn_idx][features], label=target.iloc[trn_idx], categorical_feature=categorical_feats)
 val_data = lgb.Dataset(train_df.iloc[val_idx][features], label=target.iloc[val_idx], categorical_feature=categorical_feats)
 num_round = 10000
params= param
clf2 = lgb.train(params, trn_data, num_round, valid_sets = [trn_data, val_data], verbose_eval=-1, early_stopping_rounds = 200)
 oof2[val_idx] = clf2.predict(train_df.iloc[val_idx][features], num_iteration=clf2.best_iteration)
 fold_importance_df2 = pd.DataFrame()
fold_importance_df2["feature"] = features
fold_importance_df2["importance"] = clf2.feature_importance()
fold_importance_df2["fold"] = fold_ + 1
feature_importance_df2 = pd.concat([feature_importance_df2, fold_importance_df2], axis=0)
 predictions2 += clf2.predict(test_df[features], num_iteration=clf2.best_iteration)/ folds.n_splits
print("CV score: {:<8.5f}".format(mean_squared_error(oof2, target)**0.5))
mae_train_lgb = mean_absolute_error(model.predict(x_train, num_iteration=model.best_iteration), y_train)
 mae_val_lgb = mean_absolute_error(model.predict(x_val, num_iteration=model.best_iteration), y_val)
 print('mae train lgb: ', mae_train_lgb)
 print('mae val lgb: ', mae_val_lgb )<choose_model_class>
submissionrfm = pd.read_csv('.. /input/elo-merchant-category-recommendation/sample_submission.csv')
 submissionrfm['target'] = predictions2
submissionrfm.to_csv('LGBMrfmscore1.csv', index=False)
def run_DNN(x_train, y_train, x_val, y_val, x_test):
NN_model = Sequential()
NN_model.add(Dense(x_train.shape[1], input_dim = x_train.shape[1], activation='relu'))
NN_model.add(Dense(136, activation='relu'))
NN_model.add(Dense(136, activation='relu'))
NN_model.add(Dense(136, activation='relu'))
NN_model.add(Dense(136, activation='relu'))
NN_model.add(Dense(1, activation='linear'))
NN_model.compile(loss='mean_absolute_error', optimizer='adam', metrics=['mean_absolute_error'])
 NN_model.summary()
checkpoint_name = 'Weights-{epoch:03d}--{val_loss:.5f}.hdf5'
checkpoint = ModelCheckpoint(checkpoint_name, monitor='val_loss', verbose = 1, save_best_only = True, mode ='auto')
 callbacks_list = [checkpoint]
NN_model.fit(x=x_train,
y=y_train,
batch_size=1000,
epochs=30,
verbose=1,
callbacks=callbacks_list,
validation_split=0.15,
validation_data=None,
shuffle=True,
class_weight=None,
sample_weight=None,
initial_epoch=0,
steps_per_epoch=None,
validation_steps=None)
 pred_test_y = NN_model.predict(x_test)
 pred_test_y = pred_test_y.reshape(-1)
 return pred_test_y, NN_model<train_model>
folds = RepeatedKFold(n_splits=5, n_repeats=2, random_state=15)
 oof3 = np.zeros(len(train_df))
predictions3 = np.zeros(len(test_df))
start = time.time()
feature_importance_df3 = pd.DataFrame()
for fold_,(trn_idx, val_idx)in enumerate(folds.split(train_df.values,target.values)) :
print("fold n°{}".format(fold_))
trn_data = lgb.Dataset(train_df.iloc[trn_idx][features], label=target.iloc[trn_idx], categorical_feature=categorical_feats)
 val_data = lgb.Dataset(train_df.iloc[val_idx][features], label=target.iloc[val_idx], categorical_feature=categorical_feats)
 num_round = 10000
params= param
clf3 = lgb.train(params, trn_data, num_round, valid_sets = [trn_data, val_data], verbose_eval=-1, early_stopping_rounds = 200)
 oof3[val_idx] = clf3.predict(train_df.iloc[val_idx][features], num_iteration=clf3.best_iteration)
 fold_importance_df3 = pd.DataFrame()
fold_importance_df3["feature"] = features
fold_importance_df3["importance"] = clf3.feature_importance()
fold_importance_df3["fold"] = fold_ + 1
feature_importance_df3 = pd.concat([feature_importance_df3, fold_importance_df3], axis=0)
 predictions3 += clf3.predict(test_df[features], num_iteration=clf3.best_iteration)/(5 * 2)
 print("CV score: {:<8.5f}".format(mean_squared_error(oof3, target)**0.5))
%%time
pred_test_DNN, model = run_DNN(x_train, y_train, x_val, y_val, x_test )<compute_test_metric>
submissiontarget = pd.read_csv('.. /input/elo-merchant-category-recommendation/sample_submission.csv')
 submissiontarget['target'] = predictions3
submissiontarget.to_csv('LGBMfinal1.csv', index=False)
mae_train_DNN = mean_absolute_error(model.predict(x_train), y_train)
 mae_val_DNN = mean_absolute_error(model.predict(x_val), y_val)
 print('mae train dnn: ', mae_train_DNN)
 print('mae val dnn: ', mae_val_DNN )<save_to_csv>
oof.to_csv("oof.csv",index=False)
 oof1.to_csv("oof1.csv",index=False)
 oof2.to_csv("oof2.csv",index=False)
 oof3.to_csv("oof3.csv",index=False )
pred_test_RF = RF.predict(x_test)
 df_test['winPlacePerc_RF'] = pred_test_RF
submission = df_test[['Id', 'winPlacePerc_RF']]
submission.to_csv('submission_RF.csv', index=False )<save_to_csv>
train_stack = np.vstack([oof,oof1,oof2,oof3] ).transpose()
test_stack = np.vstack([predictions, predictions1, predictions2,predictions3] ).transpose()
folds_stack = RepeatedKFold(n_splits=5, n_repeats=1, random_state=4590)
 oof_stack = np.zeros(train_stack.shape[0])
 predictions4 = np.zeros(test_stack.shape[0])
 for fold_,(trn_idx, val_idx)in enumerate(folds_stack.split(train_stack,target)) :
print("fold {}".format(fold_))
trn_data, trn_y = train_stack[trn_idx], target.iloc[trn_idx].values
val_data, val_y = train_stack[val_idx], target.iloc[val_idx].values
clf4 = BayesianRidge()
clf4.fit(trn_data, trn_y)
 oof_stack[val_idx] = clf4.predict(val_data)
 predictions4 += clf4.predict(test_stack)/ 5
np.sqrt(mean_squared_error(target.values, oof_stack))
<save_to_csv><EOS> 
sample_submission = pd.read_csv('.. /input/elo-merchant-category-recommendation/sample_submission.csv')
 sample_submission['target'] = predictions4
sample_submission.to_csv('LGBMstacksubmissionfinal1.csv', index=False )