Kaggle competition: Santander Customer Satisfaction
<SOS> metric: AUC Kaggle data source: santander-customer-satisfaction<save_to_csv>
warnings.filterwarnings('ignore')
actual_test = pd.read_csv(".. /input/titanic/test.csv")
 for_submission = pd.DataFrame({"PassengerId": actual_test['PassengerId'],
"Survived":predicted_on_actual.astype(int)})
 for_submission.to_csv("prediction_file_by_arunjith.csv",index=False )<set_options>
train = pd.read_csv(".. /input/santander-customer-satisfaction/train.csv")
 test = pd.read_csv(".. /input/santander-customer-satisfaction/test.csv")
 submission_example = pd.read_csv(".. /input/santander-customer-satisfaction/sample_submission.csv")
 print("Train dataset:", len(train), "rows and", len(train.columns), "columns")
 print("Test dataset:", len(test), "rows and", len(test.columns), "columns")
 display(train.head(5))
display(train.describe() )
%matplotlib inline
warnings.filterwarnings("ignore")
 seed = 42<load_from_csv>
print("Number of mssing data in the dataset: ", train.isna().any().sum() )
df_train = pd.read_csv('.. /input/train.csv')
 df_test = pd.read_csv('.. /input/test.csv' )<count_values>
def split_dataset(data, split_size):
y = data['TARGET']
X = data.drop(['TARGET'], axis=1)
 X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=split_size, random_state=21)
 return X_train, X_valid, y_train, y_valid
X_train, X_valid, y_train, y_valid = split_dataset(train, 0.2 )
counter = Counter(y_train)
 counter<feature_engineering>
ts = time.time()
LR = LogisticRegression()
scoring = ['accuracy', 'precision_macro', 'recall_macro' , 'f1_weighted', 'roc_auc']
scores = cross_validate(LR, X_train, y_train, scoring=scoring, cv=5)
 sorted(scores.keys())
 LR_fit_time = scores['fit_time'].mean()
LR_score_time = scores['score_time'].mean()
LR_accuracy = scores['test_accuracy'].mean()
LR_precision = scores['test_precision_macro'].mean()
LR_recall = scores['test_recall_macro'].mean()
LR_f1 = scores['test_f1_weighted'].mean()
LR_roc = scores['test_roc_auc'].mean()
print("Time spent: ", time.time() -ts )
X_train = X_train / 255
X_test = X_test / 255<concatenate>
ts = time.time()
SVM = SVC(probability = True)
 scoring = ['accuracy','precision_macro', 'recall_macro' , 'f1_weighted', 'roc_auc']
scores = cross_validate(SVM, X_train, y_train, scoring=scoring, cv=5)
 sorted(scores.keys())
 SVM_fit_time = scores['fit_time'].mean()
SVM_score_time = scores['score_time'].mean()
SVM_accuracy = scores['test_accuracy'].mean()
SVM_precision = scores['test_precision_macro'].mean()
SVM_recall = scores['test_recall_macro'].mean()
SVM_f1 = scores['test_f1_weighted'].mean()
SVM_roc = scores['test_roc_auc'].mean()
print("Time spent: ", time.time() -ts )
y_train = to_categorical(y_train, num_classes = 10)
 y_train.shape<choose_model_class>
ts = time.time()
LDA = LinearDiscriminantAnalysis()
scoring = ['accuracy', 'precision_macro', 'recall_macro' , 'f1_weighted', 'roc_auc']
scores = cross_validate(LDA, X_train, y_train, scoring=scoring, cv=5)
 sorted(scores.keys())
 LDA_fit_time = scores['fit_time'].mean()
LDA_score_time = scores['score_time'].mean()
LDA_accuracy = scores['test_accuracy'].mean()
LDA_precision = scores['test_precision_macro'].mean()
LDA_recall = scores['test_recall_macro'].mean()
LDA_f1 = scores['test_f1_weighted'].mean()
LDA_roc = scores['test_roc_auc'].mean()
print("Time spent: ", time.time() -ts )
def baseline_model() :
baseline = Sequential()
baseline.add(Conv2D(filters = 32, kernel_size =(5,5), padding = 'Same', activation ='relu',
input_shape =(28, 28, 1)))
 baseline.add(BatchNormalization())
 baseline.add(Conv2D(filters = 32, kernel_size =(5,5), padding = 'Same', activation ='relu'))
baseline.add(BatchNormalization())
 baseline.add(Conv2D(filters = 32, kernel_size =(5,5), padding = 'Same', activation ='relu'))
baseline.add(BatchNormalization())
 baseline.add(MaxPool2D(pool_size=(2,2)))
 baseline.add(Dropout(0.25))
baseline.add(Conv2D(filters = 64, kernel_size =(3,3), padding = 'Same', activation ='relu'))
baseline.add(BatchNormalization())
 baseline.add(Conv2D(filters = 64, kernel_size =(3,3), padding = 'Same', activation ='relu'))
baseline.add(BatchNormalization())
 baseline.add(Conv2D(filters = 64, kernel_size =(3,3), padding = 'Same', activation ='relu'))
baseline.add(BatchNormalization())
 baseline.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))
 baseline.add(Dropout(0.25))
baseline.add(Conv2D(filters = 128, kernel_size =(3,3), padding = 'Same', activation ='sigmoid'))
baseline.add(BatchNormalization())
 baseline.add(Flatten())
 baseline.add(Dense(128, activation = "relu"))
baseline.add(Dropout(0.4))
baseline.add(Dense(10, activation = "softmax"))
baseline.compile(optimizer = 'Adamax' , loss = "categorical_crossentropy", metrics=["accuracy"])
 return baseline<choose_model_class>
ts = time.time()
QDA = QuadraticDiscriminantAnalysis()
scoring = ['accuracy', 'precision_macro', 'recall_macro' , 'f1_weighted', 'roc_auc']
scores = cross_validate(QDA, X_train, y_train, scoring=scoring, cv=5)
 sorted(scores.keys())
 QDA_fit_time = scores['fit_time'].mean()
QDA_score_time = scores['score_time'].mean()
QDA_accuracy = scores['test_accuracy'].mean()
QDA_precision = scores['test_precision_macro'].mean()
QDA_recall = scores['test_recall_macro'].mean()
QDA_f1 = scores['test_f1_weighted'].mean()
QDA_roc = scores['test_roc_auc'].mean()
print("Time spent: ", time.time() -ts )
lr_reduction = ReduceLROnPlateau(monitor='val_acc',
patience=3,
verbose=0,
factor=0.5,
min_lr=0.00001 )<choose_model_class>
ts = time.time()
random_forest = RandomForestClassifier()
scoring = ['accuracy', 'precision_macro', 'recall_macro' , 'f1_weighted', 'roc_auc']
scores = cross_validate(random_forest, X_train, y_train, scoring=scoring, cv=5)
 sorted(scores.keys())
 forest_fit_time = scores['fit_time'].mean()
forest_score_time = scores['score_time'].mean()
forest_accuracy = scores['test_accuracy'].mean()
forest_precision = scores['test_precision_macro'].mean()
forest_recall = scores['test_recall_macro'].mean()
forest_f1 = scores['test_f1_weighted'].mean()
forest_roc = scores['test_roc_auc'].mean()
print("Time spent: ", time.time() -ts )
generator = ImageDataGenerator(featurewise_center = False,
samplewise_center = False,
featurewise_std_normalization = False,
samplewise_std_normalization = False,
zca_whitening = False,
rotation_range = 10,
zoom_range = 0.10,
width_shift_range = 0.10,
height_shift_range = 0.10,
horizontal_flip = False,
vertical_flip = False)
 generator.fit(X_train )<define_variables>
ts = time.time()
KNN = KNeighborsClassifier()
scoring = ['accuracy', 'precision_macro', 'recall_macro' , 'f1_weighted', 'roc_auc']
scores = cross_validate(KNN, X_train, y_train, scoring=scoring, cv=5)
 sorted(scores.keys())
 KNN_fit_time = scores['fit_time'].mean()
KNN_score_time = scores['score_time'].mean()
KNN_accuracy = scores['test_accuracy'].mean()
KNN_precision = scores['test_precision_macro'].mean()
KNN_recall = scores['test_recall_macro'].mean()
KNN_f1 = scores['test_f1_weighted'].mean()
KNN_roc = scores['test_roc_auc'].mean()
print("Time spent: ", time.time() -ts )
nets = 10
digits = [0] * nets
history = [0] * nets
epochs = 40
batch_size = 90<train_model>
ts = time.time()
bayes = GaussianNB()
scoring = ['accuracy', 'precision_macro', 'recall_macro' , 'f1_weighted', 'roc_auc']
scores = cross_validate(bayes, X_train, y_train, scoring=scoring, cv=5)
 sorted(scores.keys())
 bayes_fit_time = scores['fit_time'].mean()
bayes_score_time = scores['score_time'].mean()
bayes_accuracy = scores['test_accuracy'].mean()
bayes_precision = scores['test_precision_macro'].mean()
bayes_recall = scores['test_recall_macro'].mean()
bayes_f1 = scores['test_f1_weighted'].mean()
bayes_roc = scores['test_roc_auc'].mean()
print("Time spent: ", time.time() -ts )
print('Creating {0} CNNs...'.format(nets))
for model in range(nets):
digits[model] = baseline_model()
X_train_aux, X_test_aux, y_train_aux, y_test_aux = train_test_split(X_train, y_train, test_size = 0.1)
 history[model] = digits[model].fit_generator(generator.flow(X_train_aux,
y_train_aux,
batch_size = batch_size),
epochs = epochs,
steps_per_epoch = X_train_aux.shape[0] // batch_size,
validation_data =(X_test_aux, y_test_aux),
callbacks=[lr_reduction],
verbose=0)
 print("CNN {0:>2d}: Epochs = {1:d}, Max.Train accuracy = {2:.5f}, Max.Validation accuracy = {3:.5f}".format(
 model + 1,
epochs,
max(history[model].history['acc']),
max(history[model].history['val_acc'])) )<define_variables>
models_initial = pd.DataFrame({
'Model' : ['Logistic Regression', 'Support Vector Machine', 'Linear Discriminant Analysis', 'Quadratic Discriminant Analysis', 'Random Forest', 'K-Nearest Neighbors', 'Bayes'],
'Fitting time': [LR_fit_time, SVM_fit_time, LDA_fit_time, QDA_fit_time, forest_fit_time, KNN_fit_time, bayes_fit_time],
'Scoring time': [LR_score_time, SVM_score_time, LDA_score_time, QDA_score_time, forest_score_time, KNN_score_time, bayes_score_time],
'Accuracy' : [LR_accuracy, SVM_accuracy, LDA_accuracy, QDA_accuracy, forest_accuracy, KNN_accuracy, bayes_accuracy],
'Precision' : [LR_precision, SVM_precision, LDA_precision, QDA_precision, forest_precision, KNN_precision, bayes_precision],
'Recall' : [LR_recall, SVM_recall, LDA_recall, QDA_recall, forest_recall, KNN_recall, bayes_recall],
'F1_score' : [LR_f1,SVM_f1, LDA_f1, QDA_f1, forest_f1, KNN_f1, bayes_f1],
'AUC_ROC' : [LR_roc, SVM_roc, LDA_roc, QDA_roc, forest_roc, KNN_roc, bayes_roc],
}, columns = ['Model', 'Fitting time', 'Scoring time', 'Accuracy', 'Precision', 'Recall', 'F1_score', 'AUC_ROC'])
 models_initial.sort_values(by='AUC_ROC', ascending=False )
label_predicted = np.zeros(( X_test.shape[0], 10))
label_predicted<predict_on_test>
def optimize_lgb(X_train, y_train, X_valid, y_valid):
n_HP_points_to_test = 100
fit_params={"early_stopping_rounds":30,
"eval_metric" : 'auc',
"eval_set" : [(X_valid, y_valid)],
'eval_names': ['valid'],
'verbose': 0,
'categorical_feature': 'auto'}
param_test ={'max_depth': [4,5,6,7],
'num_leaves': sp_randint(6, 50),
'min_child_samples': sp_randint(100, 500),
'min_child_weight': [1e-5, 1e-3, 1e-2, 1e-1, 1, 1e1, 1e2, 1e3, 1e4],
'subsample': sp_uniform(loc=0.2, scale=0.8),
'colsample_bytree': sp_uniform(loc=0.4, scale=0.6),
'reg_alpha': [0, 1e-1, 1, 2, 5, 7, 10, 50, 100],
'reg_lambda': [0, 1e-1, 1, 5, 10, 20, 50, 100]}
clf = lgb.LGBMClassifier(max_depth=-1, random_state=21, silent=True, metric='auc', n_jobs=4, n_estimators=1000)
 gs = RandomizedSearchCV(
 estimator=clf, param_distributions=param_test,
n_iter=n_HP_points_to_test,
scoring='roc_auc',
cv=5,
refit=True,
random_state=21,
verbose=0)
 gs.fit(X_train, y_train, **fit_params)
 print('Best score reached: {} with params: {} '.format(gs.best_score_, gs.best_params_))
return gs, gs.best_score_, gs.best_params_, fit_params
X_train, X_valid, y_train, y_valid = split_dataset(train, 0.2)
 clf_1, best_score_1, optimal_params_1, fit_params = optimize_lgb(X_train, y_train, X_valid, y_valid )
for model in range(nets):
label_predicted = label_predicted + digits[model].predict(X_test )<prepare_output>
def clean_duplicates(df1, df2):
remove = []
cols = df1.columns
for i in range(len(cols)-1):
v = df1[cols[i]].values
for j in range(i+1,len(cols)) :
if np.array_equal(v,df1[cols[j]].values):
remove.append(cols[j])
 df1.drop(remove, axis=1, inplace=True)
 df2.drop(remove, axis=1, inplace=True)
 return df1, df2
def clean_constant_columns(df1, df2, threshold):
constant_cols = []
for i in df1.columns:
counts = df1[i].value_counts()
zeros = counts.iloc[0]
if zeros / len(df1)* 100 > threshold:
constant_cols.append(i)
 df1 = df1.drop(constant_cols, axis=1)
 df2 = df2.drop(constant_cols, axis=1)
 return df1, df2
train_df = train.copy()
test_df = test.copy()
train_df.var3.replace(-999999, -0.5, inplace=True)
 test_df.var3.replace(-999999, -0.5, inplace=True)
 train_id = train_df['ID']
test_id = test['ID']
train_df.drop('ID', axis=1, inplace=True)
 test_df.drop('ID', axis=1, inplace=True)
 train_df, test_df = clean_duplicates(train_df, test_df)
 train_df, test_df = clean_constant_columns(train_df, test_df, 99.9)
 X_train, X_valid, y_train, y_valid = split_dataset(train_df, 0.20)
 clf_2, best_score_2, optimal_params_2, fit_params = optimize_lgb(X_train, y_train, X_valid, y_valid )
label_predicted = np.argmax(label_predicted, axis = 1)
 label_predicted = pd.Series(label_predicted, name = "Label" )<save_to_csv>
def fix_skewness(df1, df2, nunique, max_skew):
numeric_cols = [cname for cname in df1.columns if df1[cname].dtype in ['int64', 'float64']]
skewed_feats = df1[numeric_cols].apply(lambda x: stats.skew(x.dropna())).sort_values(ascending=False)
 log_col = []
for col in skewed_feats.index:
if(df1[col].nunique() >nunique):
if(abs(skewed_feats[col])>max_skew):
if(df1[col].min() >=-0.99):
log_col.append(col)
 df1[col]=df1[col].apply(lambda x: np.log1p(x))
df2[col]=df2[col].apply(lambda x: np.log1p(x))
return df1, df2, log_col
def var38_flag(df1, df2):
df1['var38_flag'], df2['var38_flag'] = 0, 0
var38_mode = df1.var38.mode()
df1.loc[df1['var38']==var38_mode[0], ['var38', 'var38_flag']] = 0, 1
df2.loc[df2['var38']==var38_mode[0], ['var38', 'var38_flag']] = 0, 1
return df1, df2
ts = time.time()
train_df_skw, test_df_skw = train_df.copy() , test_df.copy()
train_df_skw, test_df_skw = var38_flag(train_df_skw, test_df_skw)
 train_df_skw, test_df_skw, cols_skw = fix_skewness(train_df_skw, test_df_skw, 50, 0.7)
 X_train_skw, X_valid_skw, y_train_skw, y_valid_skw = split_dataset(train_df_skw, 0.20)
 clf_3, best_score_3, optimal_params_3, fit_params = optimize_lgb(X_train_skw, y_train_skw, X_valid_skw, y_valid_skw)
 print("Time spent: ", time.time() -ts )
solution = pd.concat([pd.Series(range(1, 28001), name = "ImageId"), label_predicted], axis = 1)
 solution.to_csv("solution_cnn_opt.csv", index=False )<import_modules>
clf_lgb_1 = lgb.LGBMClassifier(colsample_bytree= 0.5041066856718041,
max_depth= 6,
min_child_samples= 215,
min_child_weight= 0.01,
num_leaves= 47,
reg_alpha= 2,
reg_lambda= 5,
subsample= 0.7631144227290101,
random_state=21,
silent=True,
metric='auc',
n_jobs=4,
n_estimators=1000)
 clf_lgb_1.fit(X_train, y_train)
 feature_imp = pd.DataFrame(sorted(zip(clf_lgb_1.feature_importances_,X_train.columns)) , columns=['Value','Feature'] ).sort_values(by=['Value'], ascending=False)[:50]
plt.figure(figsize=(20, 10))
sns.barplot(x="Value", y="Feature", data=feature_imp.sort_values(by="Value", ascending=False))
plt.title('LightGB feature importance')
 plt.tight_layout()
plt.savefig('lgb_feature_importance.png')
 plt.show()
import numpy as np
import pandas as pd
import seaborn as sns<load_from_csv>
perm = PermutationImportance(clf_lgb_1, random_state=21 ).fit(X_valid, y_valid)
 eli5.show_weights(perm, feature_names = X_valid.columns.tolist() )
<concatenate><EOS> 
clf_lgb = lgb.LGBMClassifier(colsample_bytree= 0.6641632439141401,
max_depth= 5,
min_child_samples= 224,
min_child_weight= 0.1,
num_leaves= 28,
reg_alpha= 2,
reg_lambda= 10,
subsample= 0.9244622777209361,
random_state=21,
silent=True,
metric='auc',
n_jobs=4,
n_estimators=1000)
 clf_lgb.fit(X_train, y_train, eval_set=[(X_valid, y_valid)], early_stopping_rounds=10)
 test_df = test_df[X_train.columns]
probs = clf_lgb.predict_proba(test_df)
 submission = pd.DataFrame({"ID":test_id, "TARGET": probs[:,1]})
 submission.to_csv("submission.csv", index=False )