Kaggle competition: Santander Customer Transaction Prediction
columns = df_num_train.columns
for i in range(len(columns)) :
median = df_train[columns[i]].median()
df_train[columns[i]].fillna(median, inplace=True)
 df_test[columns[i]].fillna(median, inplace=True)
 df_num_train = df_train[num_columns]
df_num_test = df_test[num_columns]<feature_engineering>
train_df = pd.read_csv('.. /input/santander-customer-transaction-prediction/train.csv')
 test_df = pd.read_csv('.. /input/santander-customer-transaction-prediction/test.csv' )
df_num_train["LotArea"] = np.log10(df_num_train["LotArea"]+1)
 df_num_train["1stFlrSF"] = np.log10(df_num_train["1stFlrSF"]+1)
 df_num_train["GrLivArea"] = np.log10(df_num_train["GrLivArea"]+1)
 df_num_test["LotArea"] = np.log10(df_num_test["LotArea"]+1)
 df_num_test["1stFlrSF"] = np.log10(df_num_test["1stFlrSF"]+1)
 df_num_test["GrLivArea"] = np.log10(df_num_test["GrLivArea"]+1 )<concatenate>
train_features = train_df.drop(['target','ID_code'], axis = 1)
 test_features = test_df.drop(['ID_code'],axis = 1)
 train_target = train_df['target']
df_cate = pd.concat([df_cate_train,df_cate_test] )<data_type_conversions>
gc.collect()
df_cate.fillna("Nan", inplace=True )<categorify>
train_all = pd.concat(( train_features,test_features),axis = 0 )
col_list = df_cate.columns
df_cate_oh = pd.DataFrame({})
 for i in col_list:
df_oh = pd.get_dummies(df_cate[i])
 df_oh.columns = df_oh.columns+str("-"+i)
 df_cate_oh = pd.concat([df_cate_oh, df_oh], axis=1)
 df_cate_oh<split>
for f in train_all.columns:
train_all[f+'_duplicate'] = train_all.duplicated(f,False ).astype(int)
df_cate_oh_train = df_cate_oh.head(len(df_cate_train))
df_cate_oh_test = df_cate_oh.tail(len(df_cate_test))<normalization>
for f in train_all.columns[0:200]:
train_all[f+'duplicate_value'] = train_all[f]*train_all[f+'_duplicate']
sc = StandardScaler()
sc.fit(df_num_train)
 df_num_train = sc.fit_transform(df_num_train)
 df_num_train = pd.DataFrame(df_num_train, columns=columns)
 df_num_test = sc.fit_transform(df_num_test)
 df_num_test = pd.DataFrame(df_num_test, columns=columns )<concatenate>
train_features = train_all.iloc[:200000]
test_features = train_all.iloc[200000:400000]
X_data = pd.concat([df_num_train, df_cate_oh_train], axis=1)
 X_test = pd.concat([df_num_test, df_cate_oh_test], axis=1 )<split>
del train_all
gc.collect()
X_train, X_val, y_train, y_val = train_test_split(X_data, y_class, test_size=0.2, random_state=10 )<train_on_grid>
n_splits = 7
splits = list(StratifiedKFold(n_splits=n_splits, shuffle=True ).split(train_features, train_target))
splits[:3]
forest = RandomForestClassifier(n_estimators=10, random_state=10)
 param_range = [10, 15, 20]
leaf = [60, 70, 80]
criterion = ["entropy", "gini", "error"]
param_grid = [{"n_estimators":param_range, "max_depth":param_range, "criterion":criterion, "max_leaf_nodes":leaf}]
gs = GridSearchCV(estimator=forest, param_grid=param_grid, scoring="accuracy", cv=10, n_jobs=1)
 gs = gs.fit(X_train, y_train)
 print(gs.best_score_)
 print(gs.best_params_ )<train_model>
cat_params = {
'learning_rate':0.01,
'max_depth':2,
'eval_metric': 'AUC',
'bootstrap_type': 'Bayesian',
'bagging_temperature': 1,
'objective': 'Logloss',
'od_type': 'Iter',
'l2_leaf_reg': 2,
'allow_writing_files': False}
forest = RandomForestClassifier(n_estimators=15, random_state=10,
criterion='gini', max_depth=10,
max_leaf_nodes=70)
 forest.fit(X_train, y_train)
 importance = forest.feature_importances_
indices = np.argsort(importance)[::-1]
for f in range(X_train.shape[1]):
print("%2d)%-*s %f" %(f+1, 30, X_data.columns[indices[f]], importance[indices[f]]))<prepare_x_and_y>
oof_cb = np.zeros(len(train_features))
predictions_cb = np.zeros(len(test_features))
for i,(train_idx, valid_idx)in enumerate(splits):
print(f'Fold {i + 1}')
 x_train = np.array(train_features)
 y_train = np.array(train_target)
 trn_x = x_train[train_idx.astype(int)]
trn_y = y_train[train_idx.astype(int)]
val_x = x_train[valid_idx.astype(int)]
val_y = y_train[valid_idx.astype(int)]
num_round = 100000
clf = CatBoostClassifier(num_round, task_type='GPU', early_stopping_rounds=1000,**cat_params,)
 clf.fit(trn_x, trn_y, eval_set=(val_x, val_y), cat_features=[], use_best_model=True, verbose=500)
 oof_cb[valid_idx] = clf.predict_proba(val_x)[:,1]
predictions_cb += clf.predict_proba(test_features)[:,1] / 5
print("CV score: {:<8.5f}".format(roc_auc_score(train_target, oof_cb)))
 gc.collect()
X = X_data[important_columns]
y = y
X_test = X_test[important_columns]<split>
param = {
'bagging_freq': 5,
'bagging_fraction': 0.33,
'boost_from_average':'false',
'boost': 'gbdt',
'feature_fraction': 0.05,
'learning_rate': 0.01,
'max_depth': -1,
'metric':'auc',
'min_data_in_leaf': 80,
'min_sum_hessian_in_leaf': 10.0,
'num_leaves': 13,
'num_threads': 12,
'tree_learner': 'serial',
'objective': 'binary',
'verbosity': 1
}
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=10 )<normalization>
oof = np.zeros(len(train_features))
predictions = np.zeros(len(test_features))
for i,(train_idx, valid_idx)in enumerate(splits):
print(f'Fold {i + 1}')
 x_train = np.array(train_features)
 y_train = np.array(train_target)
 trn_data = lgb.Dataset(x_train[train_idx.astype(int)], label=y_train[train_idx.astype(int)])
 val_data = lgb.Dataset(x_train[valid_idx.astype(int)], label=y_train[valid_idx.astype(int)])
 num_round = 100000
clf = lgb.train(param, trn_data, num_round, valid_sets = [trn_data, val_data], verbose_eval=1000, early_stopping_rounds = 3000)
 oof[valid_idx] = clf.predict(x_train[valid_idx], num_iteration=clf.best_iteration)
 predictions += clf.predict(test_features, num_iteration=clf.best_iteration)/ 5
print("CV score: {:<8.5f}".format(roc_auc_score(train_target, oof)))
 gc.collect()
tsne = TSNE(n_components=2, random_state=10)
 tsne.fit(X_train)
 tsne_X_train = tsne.fit_transform(X_train)
 tsne_X_val = tsne.fit_transform(X_val )<train_on_grid>
esemble_lgbm_cat = 0.5*oof_cb+0.5*oof
print('LightBGM auc = {:<8.5f}'.format(roc_auc_score(train_target, oof)))
 print('catboost auc = {:<8.5f}'.format(roc_auc_score(train_target, oof_cb)))
 print('LightBGM+catboost auc = {:<8.5f}'.format(roc_auc_score(train_target, esemble_lgbm_cat)) )
forest = RandomForestRegressor(n_estimators=100, random_state=10)
 params = {"max_depth":[20,21,22,23], "n_estimators":[39, 41, 43, 45]}
cv_f = GridSearchCV(forest, params, cv = 10, n_jobs =10)
 cv_f.fit(X_train, y_train)
 print("Best params:{}".format(cv_f.best_params_))
best_f = cv_f.best_estimator_
y_train_pred_f = best_f.predict(X_train)
 y_val_pred_f = best_f.predict(X_val)
 print("MSE train:{}".format(mean_squared_error(y_train, y_train_pred_f)))
 print("MSE test;{}".format(mean_squared_error(y_val, y_val_pred_f)))
 print("R2 score train:{}".format(r2_score(y_train, y_train_pred_f)))
 print("R2 score test:{}".format(r2_score(y_val, y_val_pred_f)) )<train_on_grid>
esemble_pred_lgbm_cat = 0.5*predictions+0.5*predictions_cb
xgbr = xgb.XGBRegressor()
params = {'learning_rate': [0.05, 0.1, 0.15], 'max_depth': [3, 5, 7],
'subsample': [0.85, 0.9, 0.95, 1], 'colsample_bytree': [0.3, 0.5, 0.8]}
cv_x = GridSearchCV(xgbr, params, cv = 10, n_jobs =1)
 cv_x.fit(X_train, y_train)
 print("Best params:{}".format(cv_x.best_params_))
best_x = cv_x.best_estimator_
y_train_pred_x = best_x.predict(X_train)
 y_val_pred_x = best_x.predict(X_val)
 y_train_pred_x = cv_x.predict(X_train)
 y_val_pred_x = cv_x.predict(X_val)
 print("MSE train:{}".format(mean_squared_error(y_train, y_train_pred_x)))
 print("MSE val;{}".format(mean_squared_error(y_val, y_val_pred_x)))
 print("R2 score train:{}".format(r2_score(y_train, y_train_pred_x)))
 print("R2 score val:{}".format(r2_score(y_val, y_val_pred_x)) )<train_on_grid>
id_code_test = test_df['ID_code']
lgbm = lgb.LGBMRegressor()
params = {'learning_rate': [0.08, 0.1, 0.12], 'max_depth': [20, 25, 30, 35]}
cv_lg = GridSearchCV(lgbm, params, cv = 10, n_jobs =1)
 cv_lg.fit(X_train, y_train)
 print("Best params:{}".format(cv_lg.best_params_))
best_lg = cv_lg.best_estimator_
y_train_pred_lg = best_lg.predict(X_train)
 y_val_pred_lg = best_lg.predict(X_val)
 y_train_pred_lg = cv_lg.predict(X_train)
 y_val_pred_lg = cv_lg.predict(X_val)
 print("MSE train:{}".format(mean_squared_error(y_train, y_train_pred_lg)))
 print("MSE val;{}".format(mean_squared_error(y_val, y_val_pred_lg)))
 print("R2 score train:{}".format(r2_score(y_train, y_train_pred_lg)))
 print("R2 score val:{}".format(r2_score(y_val, y_val_pred_lg)) )<train_on_grid>
my_submission_lbgm = pd.DataFrame({"ID_code" : id_code_test, "target" : predictions})
 my_submission_cat = pd.DataFrame({"ID_code" : id_code_test, "target" : predictions_cb})
 my_submission_esemble_lgbm_cat = pd.DataFrame({"ID_code" : id_code_test, "target" : esemble_pred_lgbm_cat} )
<train_on_grid><EOS> 
my_submission_lbgm.to_csv('submission_lbgm.csv', index = False, header = True)
 my_submission_cat.to_csv('submission_cb.csv', index = False, header = True)
 my_submission_esemble_lgbm_cat.to_csv('my_submission_esemble_lgbm_cat.csv', index = False, header = True )