Kaggle competition: Santander Customer Transaction Prediction
<SOS> metric: AUC Kaggle data source: santander-customer-transaction-prediction<define_variables>
pd.set_option('max_columns', None)
 %matplotlib inline
py.init_notebook_mode(connected=True)
 warnings.filterwarnings('ignore')
features_to_keep_1=[]
features_to_keep_1+=['item_category_id']
features_to_keep_1+=['item_category_freq']
features_to_keep_1+=['item_supercategory_freq']
features_to_keep_1+=['item_category_console_freq']
features_to_keep_1+=['item_category_digital_freq']
features_to_keep_1+=['shop_months_since_opening']
features_to_keep_1+=['shop_opening']
features_to_keep_1+=['item_freq_in_seniority']
features_to_keep_1+=['item_months_since_release']
features_to_keep_1+=['item_price_median_absolute_mean']
features_to_keep_1+=['item_price_median_last_sale']
features_to_keep_1+=['item_price_min_absolute_min']
features_to_keep_1+=['item_price_max_absolute_max']
features_to_keep_1+=['category_price_median_recent_mean']
features_to_keep_1+=['item_max_quantity_absolute_max']
features_to_keep_1+=['item_avg_sales_recent_mean']
features_to_keep_1+=['item_avg_sales_lag_1']
features_to_keep_1+=['item_avg_sales_last_sale']
features_to_keep_1+=['item_avg_sales_over_sold_recent_mean']
features_to_keep_1+=['item_avg_sales_over_sold_lag_1']
features_to_keep_1+=['item_avg_sales_over_sold_last_sale']
features_to_keep_1+=['item_avg_sales_seniority_1_recent_mean']
features_to_keep_1+=['item_avg_sales_seniority_1_lag_1']
features_to_keep_1+=['item_avg_sales_seniority_1_last_sale']
features_to_keep_1+=['shop_avg_sales_seniority_1_lag_12']
features_to_keep_1+=['shop_avg_sales_lag_12']
features_to_keep_1+=['category_months_since_release_avg_sales_seniority_1_absolute_mean']
features_to_keep_1+=['category_months_since_release_avg_sales_seniority_1_annual_mean']
features_to_keep_1+=['category_months_since_release_avg_sales_seniority_1_recent_mean']
features_to_keep_1+=['category_months_since_release_avg_sales_seniority_1_lag_12']
features_to_keep_1+=['shop_category_months_since_release_avg_sales_seniority_1_absolute_mean']
features_to_keep_1+=['shop_category_months_since_release_avg_sales_seniority_1_annual_mean']
features_to_keep_1+=['shop_category_months_since_release_avg_sales_seniority_1_recent_mean']
features_to_keep_1+=['shop_category_months_since_release_avg_sales_seniority_1_lag_12']
features_to_keep_1+=['supercategory_months_since_release_avg_sales_seniority_1_absolute_mean']
features_to_keep_1+=['supercategory_months_since_release_avg_sales_seniority_1_annual_mean']
features_to_keep_1+=['supercategory_months_since_release_avg_sales_seniority_1_recent_mean']
features_to_keep_1+=['supercategory_months_since_release_avg_sales_seniority_1_lag_12']
features_to_keep_1+=['shop_supercategory_months_since_release_avg_sales_seniority_1_absolute_mean']
features_to_keep_1+=['shop_supercategory_months_since_release_avg_sales_seniority_1_annual_mean']
features_to_keep_1+=['shop_supercategory_months_since_release_avg_sales_seniority_1_recent_mean']
features_to_keep_1+=['shop_supercategory_months_since_release_avg_sales_seniority_1_lag_12']
features_to_keep_1+=['shop_category_avg_sales_compared_to_category_annual_mean']
features_to_keep_1+=['shop_category_avg_sales_compared_to_category_recent_mean']
features_to_keep_1+=['shop_category_avg_sales_compared_to_category_lag_1']
features_to_keep_1+=['shop_category_avg_sales_compared_to_category_lag_12']
features_to_keep_1+=['shop_avg_sales_compare_lag12_to_annual_mean']
features_to_keep_1+=['supercategory_avg_sales_compare_recent_mean_to_annual_mean']
features_to_keep_1+=['supercategory_avg_sales_compare_lag12_to_annual_mean']
features_to_keep_1+=['category_avg_sales_compare_recent_mean_to_annual_mean']
features_to_keep_1+=['category_avg_sales_compare_lag12_to_annual_mean']
features_to_keep_1+=['category_avg_sales_compared_to_supercategory_recent_mean']
features_to_keep_1+=['category_avg_sales_compared_to_supercategory_lag_12']
features_to_keep_1+=['supercategory_month_of_release_avg_sales_compared_to_supercategory_absolute_mean']
features_to_keep_1+=['category_month_of_release_avg_sales_compared_to_category_absolute_mean']<split>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 train.shape, test.shape
features_to_discard_1=list(set(list(train_X.columns)) - set(['month_id','item_quantity']+features_to_keep_1))
train_X.drop(features_to_discard_1,axis=1,inplace=True)
 train_X.fillna(-1,inplace=True)
(X_train,Y_train,X_val,Y_val)= datasplit_train_val(train_X,n_months_val,month_id_first)
 eval_set = [(X_train,Y_train),(X_val,Y_val)]+datasplit_evalset(train_X,month_id_first)
 print('number of features to keep : '+str(len(features_to_keep_1)))
 print('number of features kept : '+str(len(X_train.columns)))
 print(X_train.info(null_counts=True,verbose=True))
del train_X
gc.collect()<init_hyperparams>
train.isnull().any().any()
xgb_params_1={'objective': 'reg:squarederror',
'max_depth': 5,
'min_child_weight': 1e3,
'subsample': 0.8,
'colsample_bytree': 0.8,
'learning_rate': 0.05,
'n_estimators': 500,
'base_score': 0.05,
'n_jobs': 4,
'random_state': 2
}
Y_train.clip(0,6,inplace=True)
 ts=time.time()
xgbreg=XGBRegressor(**xgb_params_1)
 xgbreg.fit(X_train,Y_train,eval_set=eval_set,eval_metric='rmse',verbose=False)
 print(time.time() -ts )<drop_column>
train['target'].value_counts(normalize=True )
del X_train, Y_train, X_val, Y_val
del eval_set
del rmse_guess_train, rmse_base_train, rmse_guess_val, rmse_base_val, rmse_guess_evalset, rmse_base_evalset
del xgbreg
gc.collect()<define_variables>
corrs = train.corr().abs().unstack().sort_values(kind="quicksort" ).reset_index()
corrs = corrs[corrs['level_0'] != corrs['level_1']]
corrs.tail(30 )
reset_variable_space<load_pretrained>
X = train.drop(['ID_code', 'target'], axis=1)
 y = train['target']
X_test = test.drop(['ID_code'], axis=1)
 n_fold = 4
folds = StratifiedKFold(n_splits=n_fold, shuffle=True, random_state=42)
 repeated_folds = RepeatedStratifiedKFold(n_splits=10, n_repeats=20, random_state=42)
dataset_name='train_2_pred'
n_months_val=2
month_id_first=18
train_X=pd.read_pickle(os.path.join(DATA_FOLDER,'training/'+dataset_name+'.pkl'))
train_X.drop(train_X.loc[train_X['month_id']==34].index,axis=0,inplace=True)
 print(train_X.info(null_counts=True,verbose=True))<split>
def train_model(X, X_test, y, params, folds, model_type='lgb', plot_feature_importance=False, averaging='usual', model=None):
oof = np.zeros(len(X))
prediction = np.zeros(len(X_test))
scores = []
feature_importance = pd.DataFrame()
for fold_n,(train_index, valid_index)in enumerate(folds.split(X, y)) :
print('Fold', fold_n, 'started at', time.ctime())
 X_train, X_valid = X.loc[train_index], X.loc[valid_index]
y_train, y_valid = y[train_index], y[valid_index]
if model_type == 'lgb':
train_data = lgb.Dataset(X_train, label=y_train)
 valid_data = lgb.Dataset(X_valid, label=y_valid)
 model = lgb.train(params,
train_data,
num_boost_round=20000,
valid_sets = [train_data, valid_data],
verbose_eval=1000,
early_stopping_rounds = 200)
 y_pred_valid = model.predict(X_valid)
 y_pred = model.predict(X_test, num_iteration=model.best_iteration)
 if model_type == 'xgb':
train_data = xgb.DMatrix(data=X_train, label=y_train, feature_names=X_train.columns)
 valid_data = xgb.DMatrix(data=X_valid, label=y_valid, feature_names=X_train.columns)
 watchlist = [(train_data, 'train'),(valid_data, 'valid_data')]
model = xgb.train(dtrain=train_data, num_boost_round=20000, evals=watchlist, early_stopping_rounds=200, verbose_eval=500, params=params)
 y_pred_valid = model.predict(xgb.DMatrix(X_valid, feature_names=X_train.columns), ntree_limit=model.best_ntree_limit)
 y_pred = model.predict(xgb.DMatrix(X_test, feature_names=X_train.columns), ntree_limit=model.best_ntree_limit)
 if model_type == 'sklearn':
model = model
model.fit(X_train, y_train)
 y_pred_valid = model.predict_proba(X_valid ).reshape(-1,)
 score = roc_auc_score(y_valid, y_pred_valid)
 y_pred = model.predict_proba(X_test)[:, 1]
if model_type == 'glm':
model = sm.GLM(y_train, X_train, family=sm.families.Binomial())
 model_results = model.fit()
model_results.predict(X_test)
 y_pred_valid = model_results.predict(X_valid ).reshape(-1,)
 score = roc_auc_score(y_valid, y_pred_valid)
 y_pred = model_results.predict(X_test)
 if model_type == 'cat':
model = CatBoostClassifier(iterations=20000, learning_rate=0.05, loss_function='Logloss', eval_metric='AUC', **params)
 model.fit(X_train, y_train, eval_set=(X_valid, y_valid), cat_features=[], use_best_model=True, verbose=False)
 y_pred_valid = model.predict_proba(X_valid)[:, 1]
y_pred = model.predict_proba(X_test)[:, 1]
oof[valid_index] = y_pred_valid.reshape(-1,)
 scores.append(roc_auc_score(y_valid, y_pred_valid))
if averaging == 'usual':
prediction += y_pred
elif averaging == 'rank':
prediction += pd.Series(y_pred ).rank().values
if model_type == 'lgb':
fold_importance = pd.DataFrame()
fold_importance["feature"] = X.columns
fold_importance["importance"] = model.feature_importance()
fold_importance["fold"] = fold_n + 1
feature_importance = pd.concat([feature_importance, fold_importance], axis=0)
 prediction /= n_fold
print('CV mean score: {0:.4f}, std: {1:.4f}.'.format(np.mean(scores), np.std(scores)))
 if model_type == 'lgb':
feature_importance["importance"] /= n_fold
if plot_feature_importance:
cols = feature_importance[["feature", "importance"]].groupby("feature" ).mean().sort_values(
 by="importance", ascending=False)[:50].index
best_features = feature_importance.loc[feature_importance.feature.isin(cols)]
plt.figure(figsize=(16, 12)) ;
sns.barplot(x="importance", y="feature", data=best_features.sort_values(by="importance", ascending=False)) ;
plt.title('LGB Features(avg over folds)');
return oof, prediction, feature_importance
return oof, prediction, scores
else:
return oof, prediction, scores
rmse_guess_train, rmse_base_train, rmse_guess_val, rmse_base_val, rmse_guess_evalset, rmse_base_evalset = target_analysis_val(train_X,n_months_val=2,month_id_first=18 )<define_variables>
params = {'num_leaves': 128,
'min_data_in_leaf': 42,
'objective': 'binary',
'max_depth': 16,
'learning_rate': 0.0123,
'boosting': 'gbdt',
'bagging_freq': 5,
'feature_fraction': 0.8201,
'bagging_seed': 11,
'reg_alpha': 1.728910519108444,
'reg_lambda': 4.9847051755586085,
'random_state': 42,
'metric': 'auc',
'verbosity': -1,
'subsample': 0.81,
'min_gain_to_split': 0.01077313523861969,
'min_child_weight': 19.428902804238373,
'num_threads': 4}
features_to_keep_2=[]
features_to_keep_2+=['item_category_id']
features_to_keep_2+=['item_category_freq']
features_to_keep_2+=['item_supercategory_freq']
features_to_keep_2+=['item_category_console_freq']
features_to_keep_2+=['category_semiannual_avg_recent_rsd']
features_to_keep_2+=['category_semiannual_avg_annual_rsd']
features_to_keep_2+=['shop_months_since_opening']
features_to_keep_2+=['item_months_since_release']
features_to_keep_2+=['item_months_since_first_sale_in_shop']
features_to_keep_2+=['item_months_since_last_sale_in_shop']
features_to_keep_2+=['item_price_median_absolute_mean']
features_to_keep_2+=['item_price_median_last_sale']
features_to_keep_2+=['item_price_min_absolute_min']
features_to_keep_2+=['item_price_max_absolute_max']
features_to_keep_2+=['item_price_median_compared_to_category_price_median_absolute_mean']
features_to_keep_2+=['item_price_median_compared_to_category_price_median_last_sale']
features_to_keep_2+=['category_price_median_recent_mean']
features_to_keep_2+=['item_quantity_annual_mean']
features_to_keep_2+=['item_quantity_semiannual_mean']
features_to_keep_2+=['item_quantity_recent_mean']
features_to_keep_2+=['item_quantity_lag_1']
features_to_keep_2+=['item_quantity_lag_12']
features_to_keep_2+=['item_quantity_extralin2']
features_to_keep_2+=['item_quantity_extralin3']
features_to_keep_2+=['item_quantity_absolute_min']
features_to_keep_2+=['item_quantity_absolute_max']
features_to_keep_2+=['item_avg_sales_recent_mean']
features_to_keep_2+=['item_avg_sales_lag_1']
features_to_keep_2+=['item_avg_sales_last_sale']
features_to_keep_2+=['item_avg_sales_over_sold_recent_mean']
features_to_keep_2+=['item_avg_sales_over_sold_lag_1']
features_to_keep_2+=['item_avg_sales_over_sold_lag_12']
features_to_keep_2+=['item_avg_sales_over_sold_last_sale']
features_to_keep_2+=['shop_avg_sales_seniority_2_lag_12']
features_to_keep_2+=['supercategory_avg_sales_seniority_2_recent_mean']
features_to_keep_2+=['supercategory_avg_sales_seniority_2_lag_1']
features_to_keep_2+=['supercategory_avg_sales_seniority_2_lag_12']
features_to_keep_2+=['shop_supercategory_avg_sales_seniority_2_lag_12']
features_to_keep_2+=['category_avg_sales_seniority_2_annual_mean']
features_to_keep_2+=['category_avg_sales_seniority_2_semiannual_mean']
features_to_keep_2+=['category_avg_sales_seniority_2_recent_mean']
features_to_keep_2+=['category_avg_sales_seniority_2_lag_1']
features_to_keep_2+=['shop_category_avg_sales_seniority_2_annual_mean']
features_to_keep_2+=['shop_category_avg_sales_seniority_2_semiannual_mean']
features_to_keep_2+=['shop_category_avg_sales_seniority_2_recent_mean']
features_to_keep_2+=['shop_category_avg_sales_seniority_2_lag_1']
features_to_keep_2+=['shop_category_avg_sales_seniority_2_lag_12']
features_to_keep_2+=['category_months_since_release_avg_sales_seniority_2_recent_mean']
features_to_keep_2+=['shop_category_months_since_release_avg_sales_seniority_2_absolute_mean']
features_to_keep_2+=['shop_category_months_since_release_avg_sales_seniority_2_lag_12']
features_to_keep_2+=['supercategory_months_since_release_avg_sales_seniority_2_lag_12']
features_to_keep_2+=['category_months_since_first_sale_in_shop_avg_sales_seniority_2_recent_mean']
features_to_keep_2+=['shop_category_months_since_first_sale_in_shop_avg_sales_seniority_2_lag_12']
features_to_keep_2+=['shop_category_avg_sales_compared_to_category_annual_mean']
features_to_keep_2+=['shop_category_avg_sales_compared_to_category_recent_mean']
features_to_keep_2+=['shop_avg_sales_compare_lag12_to_annual_mean']
features_to_keep_2+=['supercategory_avg_sales_compare_recent_mean_to_annual_mean']
features_to_keep_2+=['supercategory_avg_sales_compare_lag12_to_annual_mean']
features_to_keep_2+=['category_avg_sales_compare_recent_mean_to_annual_mean']
features_to_keep_2+=['category_avg_sales_compare_lag12_to_annual_mean']
features_to_keep_2+=['category_avg_sales_compared_to_supercategory_recent_mean']
features_to_keep_2+=['category_avg_sales_compared_to_supercategory_lag_12']
features_to_keep_2+=['shop_avg_sales_seniority_2_compare_lag12_to_annual_mean']
features_to_keep_2+=['supercategory_avg_sales_seniority_2_compare_lag12_to_annual_mean']
features_to_keep_2+=['category_avg_sales_seniority_2_compare_lag12_to_annual_mean']
features_to_keep_2+=['supercategory_month_of_release_avg_sales_compared_to_supercategory_absolute_mean']
features_to_keep_2+=['category_month_of_release_avg_sales_compared_to_category_absolute_mean']<split>
model = lgb.LGBMClassifier(**params, n_estimators = 20000, n_jobs = -1)
 X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.2, stratify=y)
 model.fit(X_train, y_train, eval_set=[(X_train, y_train),(X_valid, y_valid)], verbose=1000, early_stopping_rounds=200 )
features_to_discard_2=list(set(list(train_X.columns)) - set(['month_id','item_quantity']+features_to_keep_2))
train_X.drop(features_to_discard_2,axis=1,inplace=True)
 train_X['item_quantity_extralin2'].fillna(train_X['item_quantity_lag_1'],inplace=True)
 train_X['item_quantity_extralin3'].fillna(train_X['item_quantity_lag_1'],inplace=True)
 train_X.fillna(-1,inplace=True)
(X_train,Y_train,X_val,Y_val)= datasplit_train_val(train_X,n_months_val,month_id_first)
 eval_set = [(X_train,Y_train),(X_val,Y_val)]+datasplit_evalset(train_X,month_id_first)
 print('number of features to keep : '+str(len(features_to_keep_2)))
 print('number of features kept : '+str(len(X_train.columns)))
 print(X_train.info(null_counts=True,verbose=True))
del train_X
gc.collect()<init_hyperparams>
eli5.show_weights(model, targets=[0, 1], feature_names=list(X_train.columns), top=40, feature_filter=lambda x: x != '<BIAS>' )
xgb_params_2={'objective': 'reg:squarederror',
'max_depth': 6,
'min_child_weight': 300,
'subsample': 0.8,
'colsample_bytree': 0.8,
'learning_rate': 0.02,
'n_estimators': 2000,
'base_score': 0.4,
'n_jobs': 4,
'random_state': 2
}
ts=time.time()
xgbreg=XGBRegressor(**xgb_params_2)
 xgbreg.fit(X_train,Y_train,eval_set=eval_set,eval_metric='rmse',verbose=False)
 print(time.time() -ts )<drop_column>
top_features = [i for i in eli5.formatters.as_dataframe.explain_weights_df(model ).feature if 'BIAS' not in i][:100]
X1 = X[top_features]
X_train, X_valid, y_train, y_valid = train_test_split(X1, y, test_size=0.2, stratify=y)
 model.fit(X_train, y_train, eval_set=[(X_train, y_train),(X_valid, y_valid)], verbose=1000, early_stopping_rounds=200 )
del X_train, Y_train, X_val, Y_val
del eval_set
del rmse_guess_train, rmse_base_train, rmse_guess_val, rmse_base_val, rmse_guess_evalset, rmse_base_evalset
del xgbreg
gc.collect()<define_variables>
def calculate_metrics(model, X_train: pd.DataFrame() = None, y_train: pd.DataFrame() = None, X_valid: pd.DataFrame() = None,
y_valid: pd.DataFrame() = None, columns: list = [])-> pd.DataFrame() :
columns = columns if len(columns)> 0 else list(X_train.columns)
 train_pred = model.predict_proba(X_train[columns])
 valid_pred = model.predict_proba(X_valid[columns])
 f1 = 0
best_t = 0
for t in np.arange(0.1, 1, 0.05):
valid_pr =(valid_pred[:, 1] > t ).astype(int)
 valid_f1 = metrics.f1_score(y_valid, valid_pr)
 if valid_f1 > f1:
f1 = valid_f1
best_t = t
t = best_t
train_pr =(train_pred[:, 1] > t ).astype(int)
 valid_pr =(valid_pred[:, 1] > t ).astype(int)
 train_f1 = metrics.f1_score(y_train, train_pr)
 valid_f1 = metrics.f1_score(y_valid, valid_pr)
 score_df = []
print(f'Best threshold: {t:.2f}.Train f1: {train_f1:.4f}.Valid f1: {valid_f1:.4f}.')
 score_df.append(['F1', np.round(train_f1, 4), np.round(valid_f1, 4)])
 train_r = metrics.recall_score(y_train, train_pr)
 valid_r = metrics.recall_score(y_valid, valid_pr)
 score_df.append(['Recall', np.round(train_r, 4), np.round(valid_r, 4)])
 train_p = metrics.precision_score(y_train, train_pr)
 valid_p = metrics.precision_score(y_valid, valid_pr)
 score_df.append(['Precision', np.round(train_p, 4), np.round(valid_p, 4)])
 train_roc = metrics.roc_auc_score(y_train, train_pred[:, 1])
 valid_roc = metrics.roc_auc_score(y_valid, valid_pred[:, 1])
 score_df.append(['ROCAUC', np.round(train_roc, 4), np.round(valid_roc, 4)])
 train_apc = metrics.average_precision_score(y_train, train_pred[:, 1])
 valid_apc = metrics.average_precision_score(y_valid, valid_pred[:, 1])
 score_df.append(['APC', np.round(train_apc, 4), np.round(valid_apc, 4)])
 print(metrics.confusion_matrix(y_valid, valid_pr))
score_df = pd.DataFrame(score_df, columns=['Metric', 'Train', 'Valid'])
 print(score_df)
 return score_df, t
reset_variable_space<define_variables>
_ = calculate_metrics(model, X_train, y_train, X_valid, y_valid )
create_directory(os.path.join(DATA_FOLDER, 'predictions'))
create_directory(os.path.join(DATA_FOLDER, 'predictions/models/'))<load_pretrained>
dataset_name='train_0_pred'
month_id_first=18
train_X=pd.read_pickle(os.path.join(DATA_FOLDER,'training/'+dataset_name+'.pkl'))
print(train_X.info(null_counts=True,verbose=True))<split>
%%time
X = train.drop(['ID_code', 'target'], axis=1)
 X_test = test.drop(['ID_code'], axis=1)
 neigh = NearestNeighbors(3, n_jobs=-1)
 neigh.fit(X)
 dists, _ = neigh.kneighbors(X, n_neighbors=3)
 mean_dist = dists.mean(axis=1)
 max_dist = dists.max(axis=1)
 min_dist = dists.min(axis=1)
 X['mean_dist'] = mean_dist
X['max_dist'] = max_dist
X['min_dist'] = min_dist
test_dists, _ = neigh.kneighbors(X_test, n_neighbors=3)
 test_mean_dist = test_dists.mean(axis=1)
 test_max_dist = test_dists.max(axis=1)
 test_min_dist = test_dists.min(axis=1)
 X_test['mean_dist'] = test_mean_dist
X_test['max_dist'] = test_max_dist
X_test['min_dist'] = test_min_dist
oof_lgb, prediction_lgb_dist, scores = train_model(X, X_test, y, params=params, folds=folds, model_type='lgb', plot_feature_importance=True)
 sub = pd.read_csv('.. /input/sample_submission.csv')
 sub['target'] = prediction_lgb_dist
sub.to_csv('lgb_dist.csv', index=False )
rmse_guess_train, rmse_base_train, rmse_guess_evalset, rmse_base_evalset = target_analysis_test(train_X,month_id_first=18 )<train_model>