Kaggle competition: Santander Customer Transaction Prediction
<SOS> metric: AUC Kaggle data source: santander-customer-transaction-prediction<prepare_x_and_y>
warnings.filterwarnings('ignore' )
dates = final_tr_data['date_block_num']
last_block = dates.max()
print('Test `date_block_num` is {0}'.format(last_block))
X_train = final_tr_data.loc[dates < last_block]
X_test = final_tr_data.loc[dates == last_block]
columns_to_delete = ['date_block_num', 'value']
X_train = X_train.drop(columns_to_delete, axis=1)
 X_test = X_test.drop(columns_to_delete, axis=1)
 X_train = X_train.values
X_test = X_test.values
y_train = final_tr_data.loc[dates < last_block, 'value'].values
y_test = final_tr_data.loc[dates == last_block, 'value'].values
X_valid_train = final_tr_data.loc[dates < last_block-1]
X_valid_test = final_tr_data.loc[dates == last_block-1]
columns_to_delete = ['date_block_num', 'value']
X_valid_train = X_valid_train.drop(columns_to_delete, axis=1)
 X_valid_test = X_valid_test.drop(columns_to_delete, axis=1)
 X_valid_train = X_valid_train.values
X_valid_test = X_valid_test.values
y_valid_train = final_tr_data.loc[dates < last_block-1, 'value'].values
y_valid_test = final_tr_data.loc[dates == last_block-1, 'value'].values
del dates
del final_tr_data
gc.collect()<train_model>
%%time
train_df = pd.read_csv(PATH+"train.csv")
 test_df = pd.read_csv(PATH+"test.csv" )
lr = LinearRegression()
lr.fit(X_train, y_train)
 pred_lr = lr.predict(X_test ).clip(0,20 )<save_to_csv>
def missing_data(data):
total = data.isnull().sum()
percent =(data.isnull().sum() /data.isnull().count() *100)
 tt = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
 types = []
for col in data.columns:
dtype = str(data[col].dtype)
 types.append(dtype)
 tt['Types'] = types
return(np.transpose(tt))
submit = pd.DataFrame({'ID':range(len(pred_lr)) , 'item_cnt_month': pred_lr})
 submit.to_csv('submit_lr.csv', index=False )<train_model>
%%time
missing_data(train_df )
def validate(estimator, X_train_, y_train_, X_val_, y_val_, grid_params):
keys = grid_params.keys()
vals = grid_params.values()
parameters = []
rmses = []
rmses_train = []
return_obj ={}
prods = product(*vals)
 for idx, instance in enumerate(prods):
print('-'*50)
 print('model {0}:'.format(idx))
model_params = dict(zip(keys, instance))
parameters.append(model_params)
 print(model_params)
 model = estimator(**model_params)
 model.fit(X_train_, y_train_)
 pred_test = model.predict(X_val_)
 mse = mean_squared_error(y_val_, pred_test)
 rmse = np.sqrt(mse)
 print('RMSE: {0}'.format(rmse))
rmses = rmses + [rmse]
best_rmse_so_far = np.min(rmses)
 print('Best rmse so far: {0}'.format(best_rmse_so_far))
best_model_params_so_far = parameters[np.argmin(rmses)]
print('Best model params so far: {0}'.format(best_model_params_so_far))
del best_rmse_so_far
del best_model_params_so_far
del pred_test
del model
gc.collect()
rmses = np.array(rmses)
 best_rmse = np.min(rmses)
 print('Best rmse: {0}'.format(best_rmse))
best_model_params = parameters[np.argmin(rmses)]
print('Best model params: {0}'.format(best_model_params))
return_obj['rmses'] = rmses
return_obj['best_rmse'] = best_rmse
return_obj['best_model_params'] = best_model_params
return return_obj<train_model>
%%time
missing_data(test_df )
alphas = [10, 100, 1000,2000, 3000, 4000, 5000]
grid_params = {'alpha':alphas}
val_res = validate(Ridge, X_valid_train, y_valid_train,
X_valid_test, y_valid_test, grid_params )<choose_model_class>
print("There are {}% target values with 1".format(100 * train_df["target"].value_counts() [1]/train_df.shape[0]))
best_alpha=3000
ridge_model = Ridge(best_alpha)
 ridge_model.fit(X_train, y_train)
 predictions = ridge_model.predict(X_test )<save_to_csv>
%%time
correlations = train_df[features].corr().abs().unstack().sort_values(kind="quicksort" ).reset_index()
correlations = correlations[correlations['level_0'] != correlations['level_1']]
correlations.head(10 )
submit = pd.DataFrame({'ID':range(len(pred_lr)) , 'item_cnt_month': pred_lr})
 submit.to_csv('submit_ridge.csv', index=False )<init_hyperparams>
%%time
features = train_df.columns.values[2:202]
unique_max_train = []
unique_max_test = []
for feature in features:
values = train_df[feature].value_counts()
unique_max_train.append([feature, values.max() , values.idxmax() ])
 values = test_df[feature].value_counts()
unique_max_test.append([feature, values.max() , values.idxmax() ] )
best_params = {'learning_rate': 0.16, 'n_estimators': 500,
'max_depth': 6, 'min_child_weight': 7,
'subsample': 0.9, 'colsample_bytree': 0.7, 'nthread': -1,
'scale_pos_weight': 1, 'random_state': 42,
'tree_method': 'gpu_hist', 'predictor': 'gpu_predictor', 'gpu_id': 0}<choose_model_class>
%%time
idx = features = train_df.columns.values[2:202]
for df in [test_df, train_df]:
df['sum'] = df[idx].sum(axis=1)
 df['min'] = df[idx].min(axis=1)
 df['max'] = df[idx].max(axis=1)
 df['mean'] = df[idx].mean(axis=1)
 df['std'] = df[idx].std(axis=1)
 df['skew'] = df[idx].skew(axis=1)
 df['kurt'] = df[idx].kurtosis(axis=1)
 df['med'] = df[idx].median(axis=1 )
model = XGBRegressor(**best_params )<train_model>
print('Train and test columns: {} {}'.format(len(train_df.columns), len(test_df.columns)) )
model = XGBRegressor(**best_params)
 model.fit(X_valid_train,
y_valid_train,
eval_metric="rmse",
eval_set=[(X_valid_test, y_valid_test)],
verbose=True,
early_stopping_rounds = 50 )<predict_on_test>
features = [c for c in train_df.columns if c not in ['ID_code', 'target']]
target = train_df['target']
pred = model.predict(X_test ).clip(0,20 )<save_to_csv>
param = {
'bagging_freq': 5,
'bagging_fraction': 0.4,
'boost_from_average':'false',
'boost': 'gbdt',
'feature_fraction': 0.05,
'learning_rate': 0.01,
'max_depth': -1,
'metric':'auc',
'min_data_in_leaf': 80,
'min_sum_hessian_in_leaf': 10.0,
'num_leaves': 13,
'num_threads': 8,
'tree_learner': 'serial',
'objective': 'binary',
'verbosity': 1
}
submit = pd.DataFrame({'ID':range(len(pred)) , 'item_cnt_month': pred})
 submit.to_csv('submit.csv', index=False )<load_pretrained>
folds = StratifiedKFold(n_splits=10, shuffle=False, random_state=44000)
 oof = np.zeros(len(train_df))
predictions = np.zeros(len(test_df))
feature_importance_df = pd.DataFrame()
for fold_,(trn_idx, val_idx)in enumerate(folds.split(train_df.values, target.values)) :
print("Fold {}".format(fold_))
trn_data = lgb.Dataset(train_df.iloc[trn_idx][features], label=target.iloc[trn_idx])
 val_data = lgb.Dataset(train_df.iloc[val_idx][features], label=target.iloc[val_idx])
 num_round = 1000000
clf = lgb.train(param, trn_data, num_round, valid_sets = [trn_data, val_data], verbose_eval=1000, early_stopping_rounds = 3000)
 oof[val_idx] = clf.predict(train_df.iloc[val_idx][features], num_iteration=clf.best_iteration)
 fold_importance_df = pd.DataFrame()
fold_importance_df["Feature"] = features
fold_importance_df["importance"] = clf.feature_importance()
fold_importance_df["fold"] = fold_ + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 predictions += clf.predict(test_df[features], num_iteration=clf.best_iteration)/ folds.n_splits
print("CV score: {:<8.5f}".format(roc_auc_score(target, oof)) )
filename = 'xgboost_model.sav'
joblib.dump(model, filename )<load_pretrained>
sub_df = pd.DataFrame({"ID_code":test_df["ID_code"].values})
 sub_df["target"] = predictions
sub_df.to_csv("submission.csv", index=False )