Kaggle competition: Santander Customer Transaction Prediction
<SOS> metric: AUC Kaggle data source: santander-customer-transaction-prediction<split>
warnings.filterwarnings('ignore' )
train_x,valid_x,train_y,valid_y=train_test_split(X,Y,test_size=0.2,stratify=Y,random_state=100 )<train_model>
%%time
train_df = pd.read_csv(PATH+"train.csv")
 test_df = pd.read_csv(PATH+"test.csv" )
print(f'Training has {train_x.shape[0]} rows and {train_x.shape[1]} columns')
 print(f'Validation has {valid_x.shape[0]} rows and {valid_x.shape[1]} columns' )<count_values>
def missing_data(data):
total = data.isnull().sum()
percent =(data.isnull().sum() /data.isnull().count() *100)
 tt = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
 types = []
for col in data.columns:
dtype = str(data[col].dtype)
 types.append(dtype)
 tt['Types'] = types
return(np.transpose(tt))
train_y.value_counts()<count_values>
%%time
missing_data(train_df )
valid_y.value_counts()<define_variables>
%%time
missing_data(test_df )
feature_name=[f for f in train_x.columns if f not in ['target']]
mean_auc=0.0
N_SPLITS=5<define_variables>
print("There are {}% target values with 1".format(100 * train_df["target"].value_counts() [1]/train_df.shape[0]))
<define_variables>
%%time
correlations = train_df[features].corr().abs().unstack().sort_values(kind="quicksort" ).reset_index()
correlations = correlations[correlations['level_0'] != correlations['level_1']]
correlations.head(10 )
rf_oof_preds=np.zeros(train_x.shape[0])
 rf_sub_preds=np.zeros(test_model.shape[0] )<choose_model_class>
%%time
features = train_df.columns.values[2:202]
unique_max_train = []
unique_max_test = []
for feature in features:
values = train_df[feature].value_counts()
unique_max_train.append([feature, values.max() , values.idxmax() ])
 values = test_df[feature].value_counts()
unique_max_test.append([feature, values.max() , values.idxmax() ] )
folds=StratifiedShuffleSplit(n_splits=5,random_state=100 )<train_model>
%%time
idx = features = train_df.columns.values[2:202]
for df in [test_df, train_df]:
df['sum'] = df[idx].sum(axis=1)
 df['min'] = df[idx].min(axis=1)
 df['max'] = df[idx].max(axis=1)
 df['mean'] = df[idx].mean(axis=1)
 df['std'] = df[idx].std(axis=1)
 df['skew'] = df[idx].skew(axis=1)
 df['kurt'] = df[idx].kurtosis(axis=1)
 df['med'] = df[idx].median(axis=1 )
auc_score=[]
importance=pd.DataFrame()
%time
for n_folds,(train_idx,valid_idx)in enumerate(folds.split(train_x,train_y)) :
x_train,x_valid=train_x.iloc[train_idx],train_x.iloc[valid_idx]
y_train,y_valid=train_y.iloc[train_idx],train_y.iloc[valid_idx]
clf=RandomForestClassifier(n_estimators=720 ,max_depth= 50,min_samples_leaf=9 ,max_features=2 ,n_jobs=-1,random_state=100,verbose=True)
 clf.fit(x_train,y_train)
 y_preds_proba=clf.predict_proba(x_valid)
 rf_oof_preds[valid_idx]=y_preds_proba[:,1]
rf_sub_preds=clf.predict_proba(test_model[feature_name])[:,1]/folds.n_splits
auc_score.append(roc_auc_score(y_valid,rf_oof_preds[valid_idx]))
print("
{} fold ROC AUC Score is : {}".format(n_folds+1,roc_auc_score(y_valid,rf_oof_preds[valid_idx])))
 importance['feature']=feature_name
importance['gini']=clf.feature_importances_
importance['fold']=n_folds+1
print("
Average ROC Score is {}",np.mean(auc_score)) 
<init_hyperparams>
print('Train and test columns: {} {}'.format(len(train_df.columns), len(test_df.columns)) )
lgb_oof_preds=np.zeros(train_x.shape[0])
 lgb_sub_preds=np.zeros(test_model.shape[0])
 auc_valid=[]
param = {
'bagging_freq': 5,
'bagging_fraction': 0.4,
'boost_from_average':'false',
'boost': 'gbdt',
'feature_fraction': 0.05,
'learning_rate': 0.01,
'max_depth': -1,
'metric':'auc',
'min_data_in_leaf': 80,
'min_sum_hessian_in_leaf': 10.0,
'num_leaves': 13,
'num_threads': 8,
'tree_learner': 'serial',
'objective': 'binary',
'verbosity': 1
}<train_model>
features = [c for c in train_df.columns if c not in ['ID_code', 'target']]
target = train_df['target']
importances=pd.DataFrame()
for fold_idx,(train_ids, valid_ids)in enumerate(folds.split(train_x,train_y)) :
trn_data = lgb.Dataset(train_x.iloc[train_ids],label=train_y.iloc[train_ids])
 val_data = lgb.Dataset(train_x.iloc[valid_ids],label=train_y.iloc[valid_ids])
 num_rounds=10000
clf = lgb.train(param, trn_data, num_rounds, valid_sets = [trn_data, val_data], verbose_eval=1000, early_stopping_rounds = 3000)
 lgb_oof_preds[valid_ids] = clf.predict(train_x.iloc[valid_ids],num_iteration=clf.best_iteration)
 imp_df = pd.DataFrame()
imp_df['feature'] = feature_name
imp_df['gain'] = clf.feature_importance()
imp_df['fold'] = fold_idx + 1
importances = pd.concat([importances, imp_df], axis=0, sort=False)
 lgb_sub_preds += clf.predict(test_model[feature_name],num_iteration=clf.best_iteration)/ folds.n_splits
gc.collect()
print("Mean AUC: %.5f" %(roc_auc_score(train_y,lgb_oof_preds)) )<save_to_csv>
param = {
'bagging_freq': 5,
'bagging_fraction': 0.4,
'boost_from_average':'false',
'boost': 'gbdt',
'feature_fraction': 0.05,
'learning_rate': 0.01,
'max_depth': -1,
'metric':'auc',
'min_data_in_leaf': 80,
'min_sum_hessian_in_leaf': 10.0,
'num_leaves': 13,
'num_threads': 8,
'tree_learner': 'serial',
'objective': 'binary',
'verbosity': 1
}
sample_submission = pd.DataFrame({"ID_code":test["ID_code"].values})
 sample_submission["target"] =(0.4*rf_sub_preds)+(0.6*lgb_sub_preds)
 sample_submission.to_csv("blend_submission.csv", index=False )<set_options>
folds = StratifiedKFold(n_splits=10, shuffle=False, random_state=44000)
 oof = np.zeros(len(train_df))
predictions = np.zeros(len(test_df))
feature_importance_df = pd.DataFrame()
for fold_,(trn_idx, val_idx)in enumerate(folds.split(train_df.values, target.values)) :
print("Fold {}".format(fold_))
trn_data = lgb.Dataset(train_df.iloc[trn_idx][features], label=target.iloc[trn_idx])
 val_data = lgb.Dataset(train_df.iloc[val_idx][features], label=target.iloc[val_idx])
 num_round = 1000000
clf = lgb.train(param, trn_data, num_round, valid_sets = [trn_data, val_data], verbose_eval=1000, early_stopping_rounds = 3000)
 oof[val_idx] = clf.predict(train_df.iloc[val_idx][features], num_iteration=clf.best_iteration)
 fold_importance_df = pd.DataFrame()
fold_importance_df["Feature"] = features
fold_importance_df["importance"] = clf.feature_importance()
fold_importance_df["fold"] = fold_ + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 predictions += clf.predict(test_df[features], num_iteration=clf.best_iteration)/ folds.n_splits
print("CV score: {:<8.5f}".format(roc_auc_score(target, oof)) )
warnings.filterwarnings('ignore')
<load_from_csv>
sub_df = pd.DataFrame({"ID_code":test_df["ID_code"].values})
 sub_df["target"] = predictions
sub_df.to_csv("submission.csv", index=False )