Kaggle competition: Santander Customer Transaction Prediction
<SOS> metric: AUC Kaggle data source: santander-customer-transaction-prediction<load_pretrained>
import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt
from sklearn.linear_model import LogisticRegression
from sklearn.neighbors import KNeighborsClassifier
from sklearn import metrics
from sklearn import preprocessing
from sklearn import svm
from sklearn.model_selection import GridSearchCV
from sklearn import metrics
import tensorflow as tf
from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis
from sklearn.gaussian_process import GaussianProcessClassifier
from sklearn.gaussian_process.kernels import RBF
from sklearn.ensemble import RandomForestClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.naive_bayes import ComplementNB
from sklearn.covariance import EllipticEnvelope
from collections import Counter
from pandas.plotting import scatter_matrix
from sklearn.decomposition import PCA
from sklearn.ensemble import AdaBoostClassifier
from sklearn.tree import DecisionTreeClassifier
import lightgbm as lgb
alles = pd.read_pickle('data/alles_final.pkl')
 alles = alles.drop(['mean_price', 'av_shop_item'], axis=1 )<define_variables>
train_orig = pd.read_csv(".. /input/train.csv")
 test_orig = pd.read_csv(".. /input/test.csv" )
rows = alles['date_block_num'] > 11<categorify>
max_corr = -1
min_corr = 1
for i in range(199):
for j in range(i + 1, 200):
current = correlations.iloc[i, j]
min_corr = min(min_corr, current)
 max_corr = max_corr if max_corr > current else current
print('Minimum correlation: ', min_corr)
 print('Maximum correlation: ', max_corr )
alles[['item_name', 'item_category', 'item_subcategory', 'city', 'shop_type']] = \
alles[['item_name', 'item_category',
'item_subcategory', 'city', 'shop_type']].apply(LabelEncoder().fit_transform )<load_pretrained>
scaler = preprocessing.StandardScaler()
scaler.fit(X)
 X = scaler.transform(X)
 X_test = scaler.transform(X_test )
new_features = pd.read_pickle('.. /input/1c-params/new_features.pkl' )<define_variables>
seed = 125
X_train, X_dev, y_train, y_dev = train_test_split(X, y, test_size=0.05, random_state=seed, stratify = y)
 print('X_train shape: ', X_train.shape)
 print('y_train shape: ', y_train.shape)
 print('X_dev shape: ', X_dev.shape)
 print('y_dev shape: ', y_dev.shape )
features = list(new_features)+ ['date_block_num']
cols = list(features)+['item_cnt_month']<prepare_x_and_y>
models = {}
data = alles.loc[rows, cols]
data = data.fillna(0 )<define_variables>
lr = LogisticRegression(random_state=0, solver='saga', C=0.1 ).fit(X_train, y_train)
 models['LogisticRegression'] = lr
train_rows = data.date_block_num < 33
test_rows = data.date_block_num == 33<prepare_x_and_y>
qd = QuadraticDiscriminantAnalysis(reg_param=0.5, tol=1e-10 ).fit(X_train, y_train)
 models['QuadraticDiscriminantAnalysis'] = qd
X_train = data[train_rows].drop(['item_cnt_month'], axis=1)
 y_train = data[train_rows]['item_cnt_month']
X_test = data[test_rows].drop(['item_cnt_month'], axis=1)
 y_test = data[test_rows]['item_cnt_month']
X_final_test = data[data.date_block_num == 34].drop(['item_cnt_month'], axis=1 )<define_variables>
rfc = RandomForestClassifier(n_estimators=100, max_depth=2, random_state=0 ).fit(X_train, y_train)
 models['RandomForestClassifier'] = qd
categorical_features =(data.dtypes[data.dtypes.apply(str ).str.startswith('int')].index ).tolist()<define_variables>
gclf = GaussianNB().fit(X_train, y_train)
 models['GaussianNB'] = qd
categorical_features = ['item_category_id',
'shop_id']<load_pretrained>
for name, model in models.items() :
y_train_predicted = model.predict_proba(X_train)
 train_score = metrics.roc_auc_score(y_train, y_train_predicted[:, 1])
 y_dev_predicted = gclf.predict_proba(X_dev)
 dev_score = metrics.roc_auc_score(y_dev, y_dev_predicted[:, 1])
 print(name, ": ")
 print("Train score: {train}, test score: {test}".format(train=train_score, test=dev_score))
params = pd.read_pickle('.. /input/1c-params/final_params.pkl' )<split>
lgb_train = lgb.Dataset(X_train, y_train)
 lgb_eval = lgb.Dataset(X_dev, y_dev)
 param = {
'boost_from_average':'false',
'boost': 'goss',
'feature_fraction': 0.1,
'learning_rate': 0.01,
'max_depth': -1,
'metric':'auc',
'min_data_in_leaf': 100,
'min_sum_hessian_in_leaf': 10.0,
'num_leaves': 20,
'num_threads': 8,
'tree_learner': 'serial',
'objective': 'binary',
'verbosity': 1
}
num_round = 150000
clf = lgb.train(param, lgb_train, num_round, valid_sets = lgb_eval, verbose_eval=100,
early_stopping_rounds = 3000 )
kf = KFold(n_splits = 5, random_state=42)
 resu1 = 0
impor1 = 0
resu3_mae=0
y_pred = 0
y_final_test = 0
stack_train = np.zeros([X_train.shape[0], ])
 models = []
for train_index, test_index in tqdm(list(kf.split(X_train, y_train))):
X_train2 = X_train.iloc[train_index, :]
y_train2 = y_train.iloc[train_index]
X_test2 = X_train.iloc[test_index, :]
y_test2 = y_train.iloc[test_index]
d_training = lgb.Dataset(X_train2, label=y_train2,
categorical_feature=categorical_features,
free_raw_data=False)
 d_test = lgb.Dataset(X_test2, label=y_test2,
categorical_feature=categorical_features,
free_raw_data=False)
 clf = lgb.train(params, train_set=d_training, num_boost_round=1000,
valid_sets=[d_training, d_test], verbose_eval=25,
early_stopping_rounds=50)
 models.append(clf)
 temp_predict = clf.predict(X_test2, num_iteration=clf.best_iteration)
 stack_train[test_index] = temp_predict
y_pred += clf.predict(X_test, num_iteration=clf.best_iteration)/5
y_final_test += clf.predict(X_final_test, num_iteration=clf.best_iteration)/5
mse = mean_squared_error(y_test2, temp_predict)
 mae = mean_absolute_error(y_test2, temp_predict)
 resu1 += mse/5
resu3_mae += mae/5
impor1 += clf.feature_importance() /5
del X_train2, y_train2, X_test2, y_test2; gc.collect()<save_to_csv>
clf.best_iteration
test_df = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/test.csv')
 submission = pd.DataFrame({
"ID": test_df.index,
"item_cnt_month": y_final_test
})
 submission.to_csv('submission.csv', index=False )<categorify>
y_train_predicted = clf.predict(X_train, num_iteration=clf.best_iteration)
 train_score = metrics.roc_auc_score(y_train, y_train_predicted)
 y_dev_predicted = clf.predict(X_dev, num_iteration=clf.best_iteration)
 dev_score = metrics.roc_auc_score(y_dev, y_dev_predicted)
 print("Train score: {train}, test score: {test}".format(train=train_score, test=dev_score))
print("Confsuion matrix(for treshold 0.5): ")
 print("Train set: ")
 print(metrics.confusion_matrix(y_train, [1 if x > 0.5 else 0 for x in y_train_predicted]))
print("Dev set: ")
 print(metrics.confusion_matrix(y_dev, [1 if x > 0.5 else 0 for x in y_dev_predicted]))
r_features = ['item_cnt_month_lag_1', 'item_category_id', 'item_subcategory',
'item_category', 'shop_type', 'city', 'shop_id', 'date_block_num']
r_cols = list(r_features)+['item_cnt_month']
data = alles.loc[rows, r_cols]
data = data.fillna(0 )<prepare_x_and_y>
y_test_predicted = clf.predict(X_test, num_iteration=clf.best_iteration )
<define_variables><EOS> 
submission = pd.DataFrame({
"ID_code": test_orig["ID_code"],
"target": y_test_predicted
})
 submission.to_csv('sample_submission.csv', index=False )