Kaggle competition: Santander Customer Transaction Prediction
<SOS> metric: AUC Kaggle data source: santander-customer-transaction-prediction<choose_model_class>
%matplotlib inline
model = Sequential()
model.add(Dense(163, input_dim=X_train.shape[1], kernel_initializer='normal', activation='linear'))
model.add(Dense(326, activation='relu'))
model.add(Dense(163, activation='relu'))
model.add(Dense(1, activation='sigmoid'))
model.summary()<train_model>
plt.rcParams["font.size"] = 30
plt.rcParams['figure.figsize'] = [12, 8]
plt.rcParams['figure.titlesize'] = 20
plt.rcParams["axes.titlesize"] = 20
plt.rcParams["axes.labelsize"] = 20
plt.rcParams["xtick.labelsize"] = 20
plt.rcParams["ytick.labelsize"] = 20
plt.rcParams["legend.fontsize"] = 10
model.compile(loss='mae', optimizer='adam', metrics=['mae'])
 model.fit(X_train,Y, epochs=1, batch_size=50 )<train_model>
BASE_DIR = "."
DEBUG = False
np.random.seed(42 )
model.compile(loss='mae', optimizer='adam', metrics=['mae'])
 model.fit(X_train,Y, epochs=2, batch_size=100 )<train_model>
df=pd.read_csv('.. /input/train.csv')
 X_test=pd.read_csv('.. /input/test.csv' )
model.fit(X_train,Y, epochs=10, batch_size=1000 )<train_model>
df.target.value_counts()
model.fit(X_train,Y, epochs=20, batch_size=10000 )<train_model>
X = df.loc[:,'var_0':]
y = df["target"]
model.fit(X_train,Y, epochs=20, batch_size=100000 )<train_model>
model.fit(X_train,Y, epochs=5, batch_size=300000 )<predict_on_test>
count_class_0, count_class_1 = df.target.value_counts()
df_class_0 = df[df['target'] == 0]
df_class_1 = df[df['target'] == 1]
y_pred = model.predict(X_test)
<prepare_x_and_y>
df_sampled.target.value_counts()
y_pred= np.reshape(y_pred, np.shape(test_preprocessed.DBNOs)[0])
<filter>
df_sampled1=df_sampled.drop('target', axis=1 )
y_pred[y_pred > 1] = 1 
<create_dataframe>
from sklearn.metrics import confusion_matrix,accuracy_score, roc_curve, auc
from sklearn import model_selection
from sklearn.model_selection import StratifiedShuffleSplit
New= pd.DataFrame()
New['groupId']=temp1<prepare_output>
X = df_sampled.loc[:,'var_0':]
y = df_sampled["target"]
New['pred']=y_pred<load_from_csv>
df_sampled["target"].value_counts()
test = pd.read_csv('.. /input/test.csv' )<prepare_output>
import skopt
from skopt import BayesSearchCV
from skopt.space import Real, Categorical, Integer
from skopt.callbacks import DeadlineStopper
from skopt.callbacks import VerboseCallback
from skopt.callbacks import DeltaXStopper
from sklearn.metrics import make_scorer
from sklearn.metrics import average_precision_score, roc_auc_score, mean_absolute_error
import pprint
New1= pd.DataFrame()
New1['Id']=test['Id']
New1['groupId']=test['groupId']
New1.head() 
<merge>
import datetime as dt
import lightgbm as lgb
from lightgbm import LGBMClassifier
from sklearn.model_selection import StratifiedShuffleSplit
results= New1.merge(New,on='groupId' )<feature_engineering>
def report_perf(optimizer, X, y, title, callbacks=None):

 start = time()
if callbacks:
optimizer.fit(X, y, callback=callbacks)
 else:
optimizer.fit(X, y)
 best_score = optimizer.best_score_
best_score_std = optimizer.cv_results_['std_test_score'][optimizer.best_index_]
best_params = optimizer.best_params_
print(( title + " took %.2f seconds, candidates checked: %d, best CV score: %.3f "
+u"\u00B1"+" %.3f")%(time() - start,
len(optimizer.cv_results_['params']),
best_score,
best_score_std))
print('Best parameters:')
 pprint.pprint(best_params)
 print()
return best_params
results= results.merge(temp2,on='groupId')
 results["N"]=1/results["NumberOfGroups"]
results["M"]=results["pred"]/results["N"]
results["M"]=round(results["M"],0)
 results["M1"]=np.floor(results["M"])
 results["M2"]=np.ceil(results["M"])
 results["N1"] = results["N"]*results["M"]
results["N2"] = results["N"]*results["M1"]
results["N3"] = results["N"]*results["M2"]
results["K"]=1/results["maxPlace"]
results["L"]=results["pred"]/results["K"]
results["L"]=round(results["L"],0)
 results["Max1"] = results["K"]*results["L"]
results["K"]=1/(results["maxPlace"]-1)
 results["L"]=results["pred"]/results["K"]
results["L"]=round(results["L"],0)
 results["Max2"] = results["K"]*results["L"]
results["K"]=1/(results["NumberOfGroups"]-1)
 results["L"]=results["pred"]/results["K"]
results["L"]=round(results["L"],0)
 results["Max3"] = results["K"]*results["L"] 
<feature_engineering>
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=0 )
results.loc[results.maxPlace==0 , 'Max2'] = 0<correct_missing_values>
avg_prec = make_scorer(average_precision_score, greater_is_better=True, needs_proba=True )
results=results.fillna(0 )<filter>
clf = lgb.LGBMClassifier(boosting_type='gbdt',
class_weight='balanced',
objective='binary',
n_jobs=1,
verbose=0)
 search_spaces = {
'learning_rate': Real(0.01, 1.0, 'log-uniform'),
'num_leaves': Integer(2, 50),
'max_depth': Integer(0, 20),
'min_child_samples': Integer(0, 200),
'max_bin': Integer(100, 100000),
'subsample': Real(0.01, 1.0, 'uniform'),
'subsample_freq': Integer(0, 10),
'colsample_bytree': Real(0.01, 1.0, 'uniform'),
'min_child_weight': Integer(0, 10),
'subsample_for_bin': Integer(100000, 500000),
'reg_lambda': Real(1e-9, 1000, 'log-uniform'),
'reg_alpha': Real(1e-9, 1.0, 'log-uniform'),
'scale_pos_weight': Real(1e-6, 500, 'log-uniform'),
'n_estimators': Integer(500, 2000)
 }
opt = BayesSearchCV(clf,
search_spaces,
scoring=avg_prec,
cv=skf,
n_iter=40,
n_jobs=-1,
return_train_score=False,
refit=True,
optimizer_kwargs={'base_estimator': 'GP'},
random_state=22)
 best_params = report_perf(opt, X, y,'LightGBM',
callbacks=[DeltaXStopper(0.001),
DeadlineStopper(60*5)])
results[results['maxPlace']==0]<save_to_csv>
Index=X_test['ID_code']
X_test=X_test.drop('ID_code', axis=1 )
my_submission = pd.DataFrame({'Id': results.Id, 'winPlacePerc': results.N1})
 my_submission.to_csv('submission.csv', index=False)
<save_to_csv>
sub_preds=opt.predict_proba(X_test)[:,1]
my_submission = pd.DataFrame({'Id': results.Id, 'winPlacePerc': results.N2})
 my_submission.to_csv('submission1.csv', index=False )<save_to_csv>
submissions=pd.DataFrame({'ID_code':Index, 'target': sub_preds} )
my_submission = pd.DataFrame({'Id': results.Id, 'winPlacePerc': results.N3})
 my_submission.to_csv('submission2.csv', index=False )<save_to_csv>
submissions.to_csv('submissions_lgb_samplingFEtuningbest1.csv', index=False )
my_submission = pd.DataFrame({'Id': results.Id, 'winPlacePerc': results.pred})
 my_submission.to_csv('submission3.csv', index=False )<save_to_csv>
val_preds = np.zeros(X.shape[0])
 sub_preds = np.zeros(X_test.shape[0])
 kf = StratifiedShuffleSplit(n_splits=10, test_size=0.2, random_state=42)
 for i,(train_index, test_index)in enumerate(kf.split(X, y)) :
X_train, y_train = X.iloc[train_index].copy() , y.iloc[train_index]
X_valid, y_valid = X.iloc[test_index].copy() , y.iloc[test_index].copy()
print("
Fold ", i)
 clf = LGBMClassifier(
 n_estimators=1704,
learning_rate=0.16624226726409647,
num_leaves=4,
colsample_bytree=.8,
subsample=.7,
subsample_freq= 7,
subsample_for_bin= 375140,
max_depth=5,
reg_alpha=1.081049236893711e-05,
reg_lambda=1.043686239159047,
min_split_gain=.01,
min_child_weight=4,
min_child_samples=22,
silent=-1,
verbose=-1, 
)
 clf.fit(X_train, y_train,
eval_set= [(X_train, y_train),(X_valid, y_valid)],
eval_metric='auc', verbose=100, early_stopping_rounds=100 
)
 val_preds[test_index] = clf.predict_proba(X_valid, num_iteration=clf.best_iteration_)[:, 1]
sub_preds += clf.predict_proba(X_test.loc[:,'var_0':], num_iteration=clf.best_iteration_)[:, 1] / kf.n_splits
my_submission = pd.DataFrame({'Id': results.Id, 'winPlacePerc': results.Max1})
 my_submission.to_csv('submission4.csv', index=False )<save_to_csv>
val_preds[test_index] = clf.predict_proba(X_valid, num_iteration=clf.best_iteration_)[:,1]
my_submission = pd.DataFrame({'Id': results.Id, 'winPlacePerc': results.Max2})
 my_submission.to_csv('submission5.csv', index=False )<save_to_csv>
predictions_lgbm_01 = np.where(val_preds[test_index] > 0.5, 1, 0 )
my_submission = pd.DataFrame({'Id': results.Id, 'winPlacePerc': results.Max3})
 my_submission.to_csv('submission5.csv', index=False )<set_options>
lgb_F1 = f1_score(y_valid, predictions_lgbm_01, average = 'weighted')
 print("The Light GBM F1 for Validation Data is", lgb_F1 )
%matplotlib inline
sb.set()<load_from_csv>
roc_auc_score(y_valid,predictions_lgbm_01 )
train = pd.read_csv('.. /input/train_V2.csv')
 test = pd.read_csv('.. /input/test_V2.csv')
 train.dropna(inplace=True )<count_values>
clf.best_score_
train['matchType'].value_counts()<count_values>
train['teamPlayers']=train.groupId.map(train.groupId.value_counts())
 test['teamPlayers']=test.groupId.map(test.groupId.value_counts())
 train['gamePlayers']=train.matchId.map(train.matchId.value_counts())
 test['gamePlayers']=test.matchId.map(test.matchId.value_counts() )<feature_engineering>
Index=X_test['ID_code']
X_test=X_test.drop('ID_code', axis=1 )
train['enemyPlayers']=train['gamePlayers']-train['teamPlayers']
test['enemyPlayers']=test['gamePlayers']-test['teamPlayers']<feature_engineering>
submissions=pd.DataFrame({'ID_code':Index, 'target': sub_preds})
train['totalDistance']=train['rideDistance']+train['swimDistance']+train['walkDistance']
test['totalDistance']=test['rideDistance']+test['swimDistance']+test['walkDistance']<feature_engineering>
submissions.to_csv('submissions_lgb_samplingFEtuningbest2.csv', index=False )
train['enemyDamage']=train['assists']+train['kills']
test['enemyDamage']=test['assists']+test['kills']<merge>
df_sampled2=df_sampled1.drop('ID_code', axis=1)
 df_sampled2.head(2 )
totalKills = train.groupby(['matchId','groupId'] ).agg({'kills': lambda x: x.sum() })
 totalKills.rename(columns={"kills": "squadKills"}, inplace=True)
 train = train.join(other=totalKills, on=['matchId', 'groupId'])
 totalKills = test.groupby(['matchId','groupId'] ).agg({'kills': lambda x: x.sum() })
 totalKills.rename(columns={"kills": "squadKills"}, inplace=True)
 test = test.join(other=totalKills, on=['matchId', 'groupId'] )<feature_engineering>
boosting = ["goss","dart"]
def kfold_lightgbm(train_df, test_df, num_folds, stratified = False, boosting = boosting[0]):
print("Starting LightGBM.Train shape: {}, test shape: {}".format(train_df.shape, test_df.shape))
if stratified:
folds = StratifiedKFold(n_splits= num_folds, shuffle=True, random_state=326)
 else:
folds = KFold(n_splits= num_folds, shuffle=True, random_state=2045)
 oof_preds = np.zeros(train_df.shape[0])
 sub_preds = np.zeros(test_df.shape[0])
 feature_importance_df = pd.DataFrame()
for n_fold,(train_idx, valid_idx)in enumerate(folds.split(train_df, df_sampled["target"])) :
train_x, train_y = train_df.iloc[train_idx], df_sampled["target"].iloc[train_idx]
valid_x, valid_y = train_df.iloc[valid_idx], df_sampled["target"].iloc[valid_idx]
lgb_train = lgb.Dataset(train_x,label=train_y,free_raw_data=False)
 lgb_test = lgb.Dataset(valid_x,label=valid_y,free_raw_data=False)
 params ={
'task': 'train',
'boosting': 'goss',
'objective': 'binary',
'metric': 'auc',
'learning_rate': 0.01,
'subsample': 0.8,
'max_depth': -1,
'top_rate': 0.9064148448434349,
'num_leaves': 32,
'min_child_weight': 41.9612869171337,
'other_rate': 0.0721768246018207,
'reg_alpha': 9.677537745007898,
'colsample_bytree': 0.5665320670155495,
'min_split_gain': 9.820197773625843,
'reg_lambda': 8.2532317400459,
'min_data_in_leaf': 21,
'verbose': -1,
'seed':int(2**n_fold),
'bagging_seed':int(2**n_fold),
'drop_seed':int(2**n_fold)
 }
reg = lgb.train(
 params,
lgb_train,
valid_sets=[lgb_train, lgb_test],
valid_names=['train', 'test'],
num_boost_round=7000,early_stopping_rounds= 200,
verbose_eval=100, 
)
 oof_preds[valid_idx] = reg.predict(valid_x, num_iteration=reg.best_iteration)
 sub_preds += reg.predict(test_df, num_iteration=reg.best_iteration)/ folds.n_splits
fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = train_x.columns
fold_importance_df["importance"] = np.log1p(reg.feature_importance(importance_type='gain', iteration=reg.best_iteration))
fold_importance_df["fold"] = n_fold + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 print('Fold %2d roc_auc_score : %.6f' %(n_fold + 1, roc_auc_score(valid_y, oof_preds[valid_idx])))
 del reg, train_x, train_y, valid_x, valid_y
display_importances(feature_importance_df)
 submission = pd.read_csv(".. /input/sample_submission.csv")
 submission['target'] = sub_preds
submission.to_csv(boosting+".csv", index=False)
 display(submission.head())
 return(submission )
train['medicKits']=train['heals']+train['boosts']
test['medicKits']=test['heals']+test['boosts']<feature_engineering>
kfold_lightgbm(df_sampled2, X_test, num_folds=5, stratified = True, boosting = boosting[0] )
<feature_engineering><EOS> 
joblib.dump(lgb, 'lgb_train model' )