Kaggle competition: Santander Customer Transaction Prediction
print('Training the Model:')
 for fold_,(trn_idx, val_idx)in enumerate(folds.split(train_df.values, target.values)) :
print("Fold idx:{}".format(fold_ + 1))
to_appends = []
for i in range(9):
np.random.seed(i)
 to_append_train = train_df.iloc[trn_idx].loc[target==1].copy().apply(np.random.permutation)
 to_appends.append(to_append_train)
 full_append = pd.concat(to_appends,axis=0)
 full_append['target'] = 1
trn_data = lgb.Dataset(pd.concat([train_df.iloc[trn_idx][features],full_append[features]], axis=0), label=pd.concat([target.iloc[trn_idx],full_append['target']],axis=0))
val_data = lgb.Dataset(train_df.iloc[val_idx][features], label=target.iloc[val_idx])
 clf = lgb.train(param, trn_data, 1000000, valid_sets = [trn_data, val_data], verbose_eval=100, early_stopping_rounds = 4000)
 oof[val_idx] = clf.predict(train_df.iloc[val_idx][features], num_iteration=clf.best_iteration)
 predictions += clf.predict(test_df[features], num_iteration=clf.best_iteration)/ folds.n_splits
print("CV score: {:<8.5f}".format(roc_auc_score(target, oof)) )<load_from_csv>
%%time
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 print(train.shape, test.shape )
train_df = pd.read_csv('.. /input/train.csv', usecols=['ID_code','target'])
 test_df = pd.read_csv('.. /input/test.csv', usecols=['ID_code','var_0'] )<save_to_csv>
null_cnt = train.isnull().sum().sort_values()
print('null count:', null_cnt[null_cnt > 0] )
sub = pd.DataFrame({"ID_code": test_df.ID_code.values})
 sub["target"] = predictions
sub.to_csv('submission_upsampled.csv', index=False )<save_to_csv>
numcols = train.drop('target',axis=1 ).select_dtypes(include='number' ).columns.values
oofs = pd.DataFrame({"ID_code": train_df.ID_code.values})
 oofs["target"] = oof
oofs.to_csv('oof_upsampled.csv', index=False )<import_modules>
all_data = train.append(test, sort=False ).reset_index(drop=True)
 del train, test
gc.collect()
all_data.head()
import numpy as np
import pandas as pd
import gc
import lightgbm as lgb
from sklearn.model_selection import StratifiedKFold
from scipy.stats import rankdata
from sklearn.metrics import roc_auc_score<load_from_csv>
constant_column = [col for col in all_data.columns if all_data[col].nunique() == 1]
print('drop columns:', constant_column)
 all_data.drop(constant_column, axis=1, inplace=True )
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 features = [x for x in train_df.columns if x.startswith("var")]<feature_engineering>
corr_matrix = all_data.corr().abs()
upper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k=1 ).astype(np.bool))
to_drop = [c for c in upper.columns if any(upper[c] > 0.95)]
del upper
drop_column = all_data.columns[to_drop]
print('drop columns:', drop_column)
for var in features:
if np.corrcoef(train_df['target'], train_df[var])[1][0] < 0:
train_df[var] = train_df[var] * -1
test_df[var] = test_df[var] * -1<count_values>
cols = [col for col in all_data.columns if col not in ['ID_code']]
for i, t in all_data.loc[:, cols].dtypes.iteritems() :
if t == object:
print(i)
 all_data[i] = pd.factorize(all_data[i])[0]
var_stats = {}
for var in features:
var_stats[var] = train_df[var].value_counts()<feature_engineering>
scaler = preprocessing.RobustScaler()
all_data.loc[:,numcols] = scaler.fit_transform(all_data[numcols] )
def logit(p):
return np.log(p)- np.log(1 - p)
 def var_to_feat(vr, var_stats, feat_id):
new_df = pd.DataFrame()
new_df["var"] = vr.values
new_df["hist"] = pd.Series(vr ).map(var_stats)
 new_df["feature_id"] = feat_id
new_df["var_rank"] = new_df["var"].rank() /200000.
 return new_df.values<feature_engineering>
mean1 = all_data[all_data['target']==1].mean()
mean0 = all_data[all_data['target']==0].mean()
pd.concat([mean1, mean0, np.abs(mean1-mean0)], axis=1 ).sort_values(by=2, ascending=False)[1:10]
TARGET = np.array(list(train_df['target'].values)* 200)
 TRAIN = []
var_mean = {}
var_var = {}
for var in features:
tmp = var_to_feat(train_df[var], var_stats[var], int(var[4:]))
var_mean[var] = np.mean(tmp[:,0])
 var_var[var] = np.var(tmp[:,0])
 tmp[:,0] =(tmp[:,0]-var_mean[var])/var_var[var]
TRAIN.append(tmp)
 TRAIN = np.vstack(TRAIN)
 target = train_df['target'].values
del train_df
_=gc.collect()
print(TRAIN.shape, len(TARGET))<load_pretrained>
X_train = all_data[all_data['target'].notnull() ].reset_index(drop=True)
 X_test = all_data[all_data['target'].isnull() ].drop(['target'], axis=1 ).reset_index(drop=True)
 del all_data
gc.collect()
X_train.drop(['ID_code'], axis=1, inplace=True)
 X_test_ID = X_test.pop('ID_code')
 Y_train = X_train.pop('target')
 print(X_train.shape, X_test.shape )
model = lgb.LGBMClassifier(**{
'learning_rate': 0.03,
'num_leaves': 31,
'max_bin': 1023,
'min_child_samples': 1000,
'feature_fraction': 1.0,
'bagging_freq': 1,
'bagging_fraction': 0.85,
'objective': 'binary',
'n_jobs': -1,
'n_estimators':200,})
 NFOLDS = 10
predtrain = np.zeros(len(TARGET))
MODELS = []
skf = StratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=11111)
 for fold_,(train_indexes, valid_indexes)in enumerate(skf.split(TRAIN, TARGET)) :
print('Fold:', fold_)
 model = model.fit(TRAIN[train_indexes], TARGET[train_indexes],
eval_set =(TRAIN[valid_indexes], TARGET[valid_indexes]),
verbose = 100,
eval_metric='auc',
early_stopping_rounds=20,
categorical_feature = [2])
 MODELS.append(model)
 predtrain[valid_indexes] = model.predict_proba(TRAIN[valid_indexes])[:,1]
pred = np.reshape(predtrain ,(200000,200), order='F')
 print(NFOLDS,'-Fold CV AUC:',roc_auc_score(target, np.mean(logit(pred),axis=1)))
 _=gc.collect()<feature_engineering>
pca = PCA()
pca.fit(X_train)
 ev_ratio = pca.explained_variance_ratio_
ev_ratio = np.hstack([0,ev_ratio.cumsum() ])
 plt.xlabel('components')
 plt.plot(ev_ratio)
 plt.show()
ypred = np.zeros(( 200000,200))
for feat,var in enumerate(features):
tmp = var_to_feat(test_df[var], var_stats[var], int(var[4:]))
tmp[:,0] =(tmp[:,0]-var_mean[var])/var_var[var]
tmp[:,1] = tmp[:,1] + 1
tmp[ np.isnan(tmp)] = 1
for model_id in range(NFOLDS):
model = MODELS[model_id]
ypred[:,feat] += model.predict_proba(tmp)[:,1] / NFOLDS
ypred = np.mean(logit(ypred), axis=1)
 sub = test_df[['ID_code']]
sub['target'] = ypred
sub['target'] = sub['target'].rank() / 200000.
 sub.to_csv('golden_sub.csv', index=False)
 print(sub.head(20))<set_options>
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
 scores = cross_val_score(pca, X_train, Y_train, cv=skf)
 print("PCA, Accuracy: %0.4f(+/- %0.4f)" %(scores.mean() , scores.std() * 2))
scores = cross_val_score(lda, X_train, Y_train, cv=skf)
 print("LDA, Accuracy: %0.4f(+/- %0.4f)" %(scores.mean() , scores.std() * 2))
scores = cross_val_score(gnb, X_train, Y_train, cv=skf)
 print("GNB, Accuracy: %0.4f(+/- %0.4f)" %(scores.mean() , scores.std() * 2))
scores = cross_val_score(bnb, X_train, Y_train, cv=skf)
 print("BNB, Accuracy: %0.4f(+/- %0.4f)" %(scores.mean() , scores.std() * 2))
scores = cross_val_score(lgr, X_train, Y_train, cv=skf)
 print("LGR, Accuracy: %0.4f(+/- %0.4f)" %(scores.mean() , scores.std() * 2))
scores = cross_val_score(sgd, X_train, Y_train, cv=skf)
 print("SGD, Accuracy: %0.4f(+/- %0.4f)" %(scores.mean() , scores.std() * 2))
warnings.simplefilter(action='ignore', category=FutureWarning)
 warnings.filterwarnings('ignore')
 plt.style.use('seaborn')
 sns.set(font_scale=1)
<load_from_csv>
for df in [X_train, X_test]:
df['sum'] = df.sum(axis=1)
 df['min'] = df.min(axis=1)
 df['max'] = df.max(axis=1)
 df['mean'] = df.mean(axis=1)
 df['std'] = df.std(axis=1)
 df['skew'] = df.skew(axis=1)
 df['kurt'] = df.kurtosis(axis=1)
 df['med'] = df.median(axis=1)
 df['var'] = df.var(axis=1)
 df['negval'] = df.apply(lambda x:(x < 0 ).astype(int ).sum() , axis=1 )
train = pd.read_csv('.. /input/santander-customer-transaction-prediction/train.csv')
 test = pd.read_csv('.. /input/santander-customer-transaction-prediction/test.csv')
 fake_ids = np.load('.. /input/fakeids/synthetic_samples_indexes.npy')
 ids = np.arange(test.shape[0])
 real_ids = list(set(ids)- set(fake_ids))
real_test = test.iloc[real_ids]
fake_test = test.iloc[fake_ids]
real_test_id = real_test.ID_code<define_variables>
n_train = pd.DataFrame()
n_test = pd.DataFrame()
n_train['_lgr'] = X_train_lgr
n_test['_lgr'] = X_test_lgr
n_train['_sgd'] = X_train_sgd
n_test['_sgd'] = X_test_sgd
n_train['_gnb'] = X_train_gnb
n_test['_gnb'] = X_test_gnb
features = [c for c in train.columns if c not in ['target', 'ID_code']]<feature_engineering>
_=
df = pd.concat([train,real_test], axis = 0)
 for feat in tqdm_notebook(features):
df[feat+'_var'] = df.groupby([feat])[feat].transform('var')
 for feat in tqdm_notebook(features):
df[feat+'plus_'] = df[feat] + df[feat+'_var']
drop_features = [c for c in df.columns if '_var' in c]
df.drop(drop_features, axis=1, inplace=True )<split>
X_train = pd.concat([X_train, n_train], axis=1)
 X_test = pd.concat([X_test, n_test], axis=1 )
train = df.iloc[:train.shape[0]]
real_test = df.iloc[train.shape[0]:]<define_variables>
print(pd.DataFrame([[val for val in dir() ], [sys.getsizeof(eval(val)) for val in dir() ]],
index=['name','size'] ).T.sort_values('size', ascending=False ).reset_index(drop=True)[:5] )
features = [c for c in train.columns if c not in ['ID_code', 'target']]
target = train['target']<init_hyperparams>
params = {
"objective" : "binary",
"metric" : "auc",
"boosting": 'gbdt',
"max_depth" : -1,
"num_leaves" : 13,
"learning_rate" : 0.01,
"bagging_freq": 5,
"bagging_fraction" : 0.4,
"feature_fraction" : 0.05,
"min_data_in_leaf": 80,
"min_sum_heassian_in_leaf": 10,
"tree_learner": "serial",
"boost_from_average": "false",
"bagging_seed" : 42,
"verbosity" : 1,
"seed": 42
}
folds = StratifiedKFold(n_splits=10)
 oof_preds = np.zeros(X_train.shape[0])
 sub_preds = np.zeros(X_test.shape[0])
 for fold_,(trn_, val_)in enumerate(folds.split(X_train, Y_train)) :
trn_x, trn_y = X_train.iloc[trn_], Y_train[trn_]
val_x, val_y = X_train.iloc[val_], Y_train[val_]
model = lgb.LGBMRegressor(**params, n_estimators=100000)
 model.fit(trn_x, trn_y, eval_set=[(val_x, val_y)], early_stopping_rounds=3000, verbose=1000)
 oof_preds[val_] = model.predict(val_x, num_iteration=model.best_iteration_)
 sub_preds += model.predict(X_test, num_iteration=model.best_iteration_)/ folds.n_splits
param = {
'bagging_freq': 5, 'bagging_fraction': 0.4, 'boost_from_average':'false',
'boost': 'gbdt', 'feature_fraction': 0.04, 'learning_rate': 0.01,
'max_depth': -1, 'metric':'auc', 'min_data_in_leaf': 80,
'num_leaves': 13, 'num_threads': 8,
'tree_learner': 'serial', 'objective': 'binary', 'verbosity': 1
}<create_dataframe>
_=
folds = StratifiedKFold(n_splits=5, shuffle=True, random_state=4242)
 oof = np.zeros(len(train))
predictions = np.zeros(len(real_test))
val_aucs = []
feature_importance_df = pd.DataFrame()<find_best_model_class>
preds = sub_preds
<compute_test_metric><EOS> 
submission = pd.DataFrame({
'ID_code': X_test_ID,
'target': preds
})
 submission.to_csv("submission.csv", index=False )