Kaggle competition: Don't Overfit! II
<SOS> metric: AUC Kaggle data source: dont-overfit!-ii<feature_engineering>
import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.preprocessing import StandardScaler
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import roc_auc_score, make_scorer
from scipy.stats import describe, rankdata, mannwhitneyu
from tqdm import tqdm_notebook
import os
df['DAYS_EMPLOYED_PERC'] = df['DAYS_EMPLOYED'] / df['DAYS_BIRTH']
df['INCOME_CREDIT_PERC'] = df['AMT_INCOME_TOTAL'] / df['AMT_CREDIT']
df['INCOME_PER_PERSON'] = df['AMT_INCOME_TOTAL'] / df['CNT_FAM_MEMBERS']
df['ANNUITY_INCOME_PERC'] = df['AMT_ANNUITY'] / df['AMT_INCOME_TOTAL']<define_variables>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )
a = df['DAYS_EMPLOYED_PERC'].tolist()
a = [x for x in a if str(x)!= 'nan']
b = df['INCOME_CREDIT_PERC'].tolist()
b = [x for x in b if str(x)!= 'nan']
c = df['INCOME_PER_PERSON'].tolist()
c = [x for x in c if str(x)!= 'nan']
d = df['ANNUITY_INCOME_PERC'].tolist()
d = [x for x in d if str(x)!= 'nan']<train_model>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )
print("End Train Test.................. " )<train_model>
corrs = train.corr().abs().unstack().sort_values(kind="quicksort" ).reset_index()
corrs = corrs[corrs['level_0'] != corrs['level_1']]
corrs.tail(10 )
print("Start Bureau................ " )<load_from_csv>
y = train['target']
train.drop('target', axis = 1, inplace = True )
bureau = pd.read_csv('.. /input/bureau.csv', nrows = num_rows )<load_from_csv>
mw = pd.DataFrame(index = ['stat', 'p'])
 for c in train.columns:
mw[c] = mannwhitneyu(train[c], test[c])
 mw = mw.T
mw[mw['p'] <.01].shape
bad_features = list(mw[mw['p'] <.01].index)
 print(bad_features )
bb = pd.read_csv('.. /input/bureau_balance.csv', nrows = num_rows)
 bb, bb_cat = one_hot_encoder(bb, nan_as_category)
 bureau, bureau_cat = one_hot_encoder(bureau, nan_as_category )<merge>
def apply_noise(data, noise_level):
idxt = data[['id', 'target']]
features = data.columns.values[2:]
appended_data = []
for feature in features:
signal = data[feature]
noise_factor =(np.abs(signal)).mean() * noise_level
noise = np.random.normal(0, noise_level, signal.shape)
 jittered = signal + noise
appended_data.append(pd.DataFrame(jittered))
appended_data = pd.concat(appended_data, axis=1)
 data_jittered = pd.concat([idxt, pd.DataFrame(appended_data)], axis=1)
 return data_jittered
noise_train_df = []
bb_aggregations = {'MONTHS_BALANCE': ['min', 'max', 'size']}
for col in bb_cat:
bb_aggregations[col] = ['mean']
bb_agg = bb.groupby('SK_ID_BUREAU' ).agg(bb_aggregations)
 bb_agg.columns = pd.Index([e[0] + "_" + e[1].upper() for e in bb_agg.columns.tolist() ])
 bureau = bureau.join(bb_agg, how='left', on='SK_ID_BUREAU')
 bureau.drop(columns= 'SK_ID_BUREAU', inplace= True)
 del bb, bb_agg
gc.collect()<define_variables>
X = train.drop(bad_features, axis=1)
num_aggregations = {
'DAYS_CREDIT': ['min', 'max', 'mean', 'var'],
'CREDIT_DAY_OVERDUE': ['max', 'mean'],
'DAYS_CREDIT_ENDDATE': ['min', 'max', 'mean'],
'AMT_CREDIT_MAX_OVERDUE': ['mean'],
'CNT_CREDIT_PROLONG': ['sum'],
'AMT_CREDIT_SUM': ['max', 'mean', 'sum'],
'AMT_CREDIT_SUM_DEBT': ['max', 'mean', 'sum'],
'AMT_CREDIT_SUM_OVERDUE': ['mean'],
'AMT_CREDIT_SUM_LIMIT': ['mean', 'sum'],
'DAYS_CREDIT_UPDATE': ['min', 'max', 'mean'],
'AMT_ANNUITY': ['max', 'mean'],
'MONTHS_BALANCE_MIN': ['min'],
'MONTHS_BALANCE_MAX': ['max'],
'MONTHS_BALANCE_SIZE': ['mean', 'sum']
}<feature_engineering>
lr = LogisticRegression(class_weight='balanced', penalty='l1', C=0.1, solver='liblinear')
 scores = cross_val_score(lr, X, y, scoring='roc_auc', cv=3)
 scores.mean()
<merge><EOS> 
lr.fit(X, y)
 sub_pred = lr.predict_proba(test.drop(bad_features, axis=1)) [:,1]
sub = pd.DataFrame({"id":test["id"].values})
 sub["target"] = sub_pred
sub.to_csv("submission_prob_whitney.csv", index=False )