Kaggle competition: Instant Gratification
<SOS> metric: AUC Kaggle data source: instant-gratification<choose_model_class>
optuna.logging.disable_default_handler()
warnings.filterwarnings("ignore")
 %matplotlib inline
pd.set_option('max_columns', 9999)
 pd.set_option('max_rows', 9999)
 start = time.time()
os.listdir('.. /input/' )
logreg = sklearn.linear_model.LogisticRegression()
extra_trees = sklearn.ensemble.ExtraTreesClassifier(max_depth = 4,n_estimators=10)
 gradient_boost = sklearn.ensemble.GradientBoostingClassifier(max_depth = 4,n_estimators=10)
 random_forest = sklearn.ensemble.RandomForestClassifier(max_depth = 4,n_estimators=10)
 decision_tree = sklearn.tree.DecisionTreeClassifier(max_depth = 4)
 gaussianNB = sklearn.naive_bayes.GaussianNB()
ada_boost = sklearn.ensemble.AdaBoostClassifier(n_estimators=10)
 base_models = {'logreg': logreg, 'extra_trees': extra_trees, 'ada_boost': ada_boost,
'gradient_boost': gradient_boost, 'random_forest': random_forest,
'decision_tree': decision_tree, 'gaussianNB': gaussianNB}
take_models = ['logreg', 'extra_trees', 'gradient_boost', 'gaussianNB',
'random_forest', 'decision_tree', 'nn', 'ada_boost']
train_acc = {}
valid_acc = {}
y_test_pred_proba = {}
y_train_pred_proba = {}
y_valid_pred_proba = {}
for mn in take_models:
train_acc[mn] = 0
valid_acc[mn] = 0
y_test_pred_proba[mn] = 0
cv_num = 10
kfold = sklearn.model_selection.KFold(cv_num, shuffle=True)
 print('Training {} models with {}-fold cross validation'.format(len(take_models), cv_num))
print('')
 for i,(train_index, valid_index)in enumerate(kfold.split(x_train_valid)) :
print('{}.training of models in progress'.format(i+1))
x_train = x_train_valid[train_index]
y_train = y_train_valid[train_index]
x_valid = x_train_valid[valid_index]
y_valid = y_train_valid[valid_index]
for mn in take_models:
if mn == 'nn':
tf_tensors = create_nn_graph(x_train_valid.shape[1], 1)
 params = train_nn_graph(tf_tensors, x_train, y_train.reshape(-1,1),
x_valid, y_valid.reshape(-1,1), False)
 train_acc['nn'] += get_accuracy(params[0], y_train)
 valid_acc['nn'] += get_accuracy(params[1], y_valid)
 y_test_pred_proba['nn'] += params[2]
else:
model = sklearn.base.clone(base_models[mn])
 model.fit(x_train, y_train)
 train_acc[mn] += get_accuracy(model.predict_proba(x_train)[:,1], y_train)
 valid_acc[mn] += get_accuracy(model.predict_proba(x_valid)[:,1], y_valid)
 y_test_pred_proba[mn] += model.predict_proba(x_test)[:,1]
print('')
 for mn in take_models:
train_acc[mn] /= cv_num
valid_acc[mn] /= cv_num
y_test_pred_proba[mn] /= cv_num
print(mn,'train/valid accuracy = %.3f/%.3f'%(train_acc[mn], valid_acc[mn]))
take_model_avg = ['logreg', 'random_forest', 'decision_tree']
train_acc['majority_vote'] = 0
valid_acc['majority_vote'] = 0
y_test_pred_proba['majority_vote'] = 0
for mn in take_model_avg:
train_acc['majority_vote'] += train_acc[mn] / len(take_model_avg)
 valid_acc['majority_vote'] += valid_acc[mn] / len(take_model_avg)
 y_test_pred_proba['majority_vote'] += y_test_pred_proba[mn] / len(take_model_avg)
 print('')
 print('Average models:', take_model_avg)
 print('Soft majority vote: train/valid accuracy = %.3f/%.3f'%(train_acc['majority_vote'],
valid_acc['majority_vote'])) 
<prepare_output>
def load_data(nrows=None, verbose=True):
train = pd.read_csv('.. /input/train.csv', nrows=nrows)
 test = pd.read_csv('.. /input/test.csv')
 if verbose:
print('Train Shape: {}'.format(train.shape))
print('Test Shape: {}'.format(test.shape))
return train, test
y_test_pred_df = pd.DataFrame({})
 for key in y_test_pred_proba.keys() :
y_test_pred_df[key] = get_classes(y_test_pred_proba[key])
 corrmat = y_test_pred_df.corr()
corrmat
plt.subplots(figsize=(10,5))
plt.title('Correlation of Test Predictions')
 sns.heatmap(corrmat, vmax=1, square=True )<train_model>
def get_mean_cov(x,y):
model = GraphicalLasso()
ones =(y==1 ).astype(bool)
 x2 = x[ones]
model.fit(x2)
 p1 = model.precision_
m1 = model.location_
onesb =(y==0 ).astype(bool)
 x2b = x[onesb]
model.fit(x2b)
 p2 = model.precision_
m2 = model.location_
ms = np.stack([m1,m2])
 ps = np.stack([p1,p2])
 return ms,ps
take_models = ['logreg', 'random_forest', 'gradient_boost', 'ada_boost', 'nn']
cv_num = 10
kfold = sklearn.model_selection.KFold(cv_num, shuffle=True)
 print('Ensemble learning with', len(take_models),'models')
 print('Pasting with', cv_num, 'out-of-fold predictions')
 print('')
 for i,(train_index, valid_index)in enumerate(kfold.split(x_train_valid)) :
x_train = x_train_valid[train_index]
y_train = y_train_valid[train_index]
x_valid = x_train_valid[valid_index]
y_valid = y_train_valid[valid_index]
for mn in take_models:
if mn == 'nn':
tf_tensors = create_nn_graph(x_train_valid.shape[1],1)
 params = train_nn_graph(tf_tensors, x_train, y_train.reshape(-1,1),
x_valid, y_valid.reshape(-1,1), False)
 y_train_pred_proba[mn,i] = params[0].reshape(-1,)
 y_valid_pred_proba[mn,i] = params[1].reshape(-1,)
 y_test_pred_proba[mn,i] = params[2].reshape(-1,)
 else:
model = sklearn.base.clone(base_models[mn])
 model.fit(x_train, y_train)
 y_train_pred_proba[mn,i] = model.predict_proba(x_train)[:,1]
y_valid_pred_proba[mn,i] = model.predict_proba(x_valid)[:,1]
y_test_pred_proba[mn,i] = model.predict_proba(x_test)[:,1]
print(i, mn,': train/valid accuracy = %.3f/%.3f'%(
 get_accuracy(y_train_pred_proba[mn,i], y_train),
get_accuracy(y_valid_pred_proba[mn,i], y_valid)))
 if i==0:
y_train_meta = y_valid
else:
y_train_meta = np.concatenate([y_train_meta, y_valid])
 if False:
for i in range(cv_num):
tmp = 0
for mn in take_models:
tmp += y_valid_pred_proba[mn,i] / len(take_models)
 x_test_meta += y_test_pred_proba[mn,i] /(len(take_models)*cv_num)
 x_train_meta = np.concatenate([x_train_meta, tmp])
 if True:
for i in range(cv_num):
for j,mn in enumerate(take_models):
if j==0:
tmp1 = y_valid_pred_proba[mn,i].reshape(-1,1)
 tmp2 = y_test_pred_proba[mn,i].reshape(-1,1)
 else:
tmp1 = np.concatenate([tmp1, y_valid_pred_proba[mn,i].reshape(-1,1)], axis=1)
 tmp2 = np.concatenate([tmp2, y_test_pred_proba[mn,i].reshape(-1,1)], axis=1)
 if i==0:
x_train_meta = tmp1
x_test_meta = tmp2 / cv_num
else:
x_train_meta = np.concatenate([x_train_meta, tmp1], axis=0)
 x_test_meta += tmp2 / cv_num
print('')
 print('Ensemble method: soft majority vote of all models')
 y_valid_pred_proba['majority_vote'] = np.mean(x_train_meta, axis=1)
 y_test_pred_proba['majority_vote'] = x_test_meta
print('Soft majority vote: valid accuracy = %.3f'%get_accuracy(y_train_meta,
y_valid_pred_proba['majority_vote']))<train_model>
def trainer(train, test, cv, params, model_type='qda', _scaler=True, SVD=None,
variance=False, _model=None, _plot=True, threshold=1.5):
_train, _test = train.copy() , test.copy()
_test['target'] = -1
cols = [col for col in _train.columns if not col in ['id', 'target', 'wheezy-copper-turtle-magic']]
preds = np.zeros(len(_test))
oof = np.zeros(len(_train))
oof_org = np.zeros(len(train_org))
for i in tqdm(range(512)) :
train_2 = _train[_train['wheezy-copper-turtle-magic'] == i]
test_2 = _test[_test['wheezy-copper-turtle-magic'] == i]
train_2_org = train_org[train_org['wheezy-copper-turtle-magic'] == i]
len_train = len(train_2)
 Y = train_2['target'].reset_index(drop=True ).values
train_index = train_2.index
test_index = test_2.index
train_org_index = train_2_org.index
merged_data = pd.concat([train_2, test_2], axis=0 ).reset_index(drop=True)[cols].values
if SVD is not None:
clf = TruncatedSVD(n_components=SVD)
 _X = clf.fit_transform(merged_data)
 X_org = clf.transform(train_2_org[cols].values)
 elif variance:
clf = VarianceThreshold(threshold=threshold)
 _X = clf.fit_transform(merged_data)
 X_org = clf.transform(train_2_org[cols].values)
 else:
_X = merged_data
X_org = clf.transform(train_2_org[cols].values)
 if _scaler:
_scaler = StandardScaler()
_X = _scaler.fit_transform(_X)
 X_org = _scaler.transform(X_org)
 X = _X[:len_train]
test_2 = _X[len_train:]
del merged_data, _X
gc.collect()
X = X.astype(np.float32)
 test_2 = test_2.astype(np.float32)
 X_org = X_org.astype(np.float32)
 for i,(trn_index, val_index)in enumerate(cv.split(X, Y)) :
if model_type == 'gmm':
ms, ps = get_mean_cov(X, Y)
 X_trn, X_val = X[trn_index], X[val_index]
Y_trn, Y_val = Y[trn_index], Y[val_index]
if model_type == 'nusvc':
model = NuSVC(**params)
 elif model_type == 'qda':
model = QuadraticDiscriminantAnalysis(**params)
 elif model_type == 'gmm':
model = GaussianMixture(**params, means_init=ms, precisions_init=ps)
 elif model_type == None:
model = _model
model.fit(X_trn, Y_trn)
 oof[train_index[val_index]] += model.predict_proba(X_val)[:,1]
preds[test_index] += model.predict_proba(test_2)[:,1] / cv.n_splits
oof_org[train_org_index] += model.predict_proba(X_org)[:,1] / cv.n_splits
if model_type == 'gmm':
oof = 1.0 - oof
preds = 1.0 - preds
oof_org = 1.0 - oof_org
auc = roc_auc_score(_train['target'], oof[:_train.shape[0]])
 auc_org = roc_auc_score(train_org['target'], oof_org)
 if _plot:
fig = plt.figure(figsize=(6, 4))
sns.distplot(np.array(preds), kde=False, bins=25)
 plt.title('Model [{}] AUC: {:.5f}'.format(model_type, auc_org))
plt.show()
return np.array(preds), oof[:train.shape[0]]
take_meta_model = 'logreg'
print('')
 print('Ensemble method: stacking of base models using the meta-model', take_meta_model)
 print('x_train_meta.shape =', x_train_meta.shape)
 print('y_train_meta.shape =', y_train_meta.shape)
 print('x_test_meta.shape =', x_test_meta.shape)
 model = sklearn.base.clone(base_models[take_meta_model])
 model.fit(x_train_meta, y_train_meta)
 y_train_pred_proba['meta_model'] = model.predict_proba(x_train_meta)[:,1]
y_test_pred_proba['meta_model'] = model.predict_proba(x_test_meta)[:,1]
print('Meta model: train accuracy = %.3f'%get_accuracy(y_train_meta,
y_train_pred_proba['meta_model']))<predict_on_test>
train, test = load_data()
train_org = train.copy()
y_test_submit = get_classes(y_test_pred_proba['meta_model'])
 y_test_submit.mean()<save_to_csv>
def PsuedoLabeling(train, test, reg_param, _min=0.01, _max=0.99):
_train = train.copy()
_test = test.copy()
qda_params = {
'reg_param': reg_param
}
cv = StratifiedKFold(n_splits=15)
 preds, oof = trainer(_train, _test, cv, qda_params, model_type='qda', variance=True, _plot=False, _scaler=False)
 _test['target'] = preds
test_0 = _test[_test['target'] < _min]
test_0['target'] = 0
test_1 = _test[_test['target'] > _max]
test_1['target'] = 1
train = pd.concat([train, test_0, test_1], axis=0 ).reset_index(drop=True)
 return train
def run_PsuedoLabeling(train, test, reg_param, _min=0.01, _max=0.99, _iter=20):
reg_param = random.uniform(0.1, 0.6)
 for _ in range(_iter):
before_train_shape = train.shape[0]
train = PsuedoLabeling(train, test, reg_param, _min=_min, _max=_max)
 train = train.drop_duplicates(keep='first' ).reset_index(drop=True)
 print('Train Data Shape: {}'.format(train.shape))
if train.shape[0] ==before_train_shape:
break
return train
submission = pd.DataFrame({
"PassengerId": test_df["PassengerId"],
"Survived": y_test_submit
})
 submission.to_csv('submission.csv', index=False)
 submission.head()<load_from_csv>
def change_label(train, test, reg_param):
modified_train = train.copy()
_train = train.copy()
_test = test.copy()
qda_params = {
'reg_param': reg_param
}
cv = StratifiedKFold(n_splits=15, random_state=46)
 preds, oof = trainer(_train, _test, cv, qda_params, model_type='qda', variance=True, _plot=False, _scaler=False)
 df = pd.DataFrame({
'id': train['id'].values,
'target': train['target'].values,
'oof': oof
})
 df_0 = df[(df['target'] == 1)&(df['oof'] < 0.01)]
df_1 = df[(df['target'] == 0)&(df['oof'] > 0.99)]
modified_train.loc[df_0.index, 'target'] = 0
modified_train.loc[df_1.index, 'target'] = 1
modified_rows = len(df_0)+ len(df_1)
 print('Modified {} rows'.format(modified_rows))
return modified_train, modified_rows
def run_change_label(train, test, reg_param, _iter=20):
reg_param = random.uniform(0.1, 0.6)
 for i in range(_iter):
train, modified_rows = change_label(train, test, reg_param)
 train.reset_index(drop=True, inplace=True)
 if modified_rows < 10:
break
return train
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )<load_from_csv>
%%time
prep_start = time.time()
train = run_PsuedoLabeling(train, test, reg_param=0.5, _min=0.04, _max=0.96, _iter=10)
 train = run_change_label(train, test, reg_param=0.5, _iter=3)
 train.to_csv('Preprocessed_train.csv' )
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )<count_missing_values>
prep_end = time.time()
elapsedtime = prep_end - prep_start
s = datetime.timedelta(seconds=elapsedtime)
 print('Preprocessing 1 epoch time: {}'.format(str(s)))
 print('Final Train Data Shape: {}'.format(train.shape))
print(train.isnull().sum() /train.shape[0] )<count_missing_values>
tr = pd.DataFrame({
'target': train['target'].values
})
 te = pd.DataFrame()
print(test.isnull().sum() /test.shape[0] )<groupby>
%%time
name = 'qda'
qda_params = {
'reg_param': 0.01849,
'priors': 17.0978,
'tol': 6.943e-06
}
cv = StratifiedKFold(n_splits=15, random_state=46, shuffle=True)
 preds, oof = trainer(train, test, cv, qda_params, model_type=name.split('_')[0],
variance=True, _scaler=True, threshold=2.99)
 tr[name] = oof
te[name] = preds
print(train.groupby(['FamilyBin', 'Pclass'])[['Survived']].mean())
 print(train.groupby(['FamilyBin', 'Pclass'])[['Survived']].count() )<groupby>
%%time
name = 'nusvc'
nusvc_params = {
'probability': True,
'kernel': 'poly',
'degree': 3,
'gamma': 'auto',
'coef0': 0.159,
'nu': 0.1486,
'tol': 1.8909e-4
}
cv = StratifiedKFold(n_splits=5, random_state=46)
 preds, oof = trainer(train, test, cv, nusvc_params, model_type=name.split('_')[0], SVD=20)
 tr[name] = oof
te[name] = preds
print(train.groupby(['Embarked', 'Pclass'])[['Survived']].count())
 print(train.groupby(['Embarked', 'Pclass'])[['Survived']].mean())
 print(train.groupby(['Embarked'])[['Pclass']].count() )<feature_engineering>
%%time
name = 'gmm'
gmm_params = {
'n_components': 2,
'init_params': 'kmeans',
'covariance_type': 'full',
'tol': 0.001,
'reg_covar': 0.000389422,
'max_iter': 536,
'n_init': 1
}
cv = StratifiedKFold(n_splits=5, random_state=46, shuffle=True)
 preds, oof = trainer(train, test, cv, gmm_params, model_type=name.split('_')[0],
variance=True, _scaler=False, threshold=2.21)
 tr[name] = oof
te[name] = preds
def titleExtract(data):
data['Title'] = data.Name.str.extract('([A-Za-z]+)\.', expand=False)
 data['Title'] = data['Title'].replace(['Capt', 'Col','Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer'], 'Mr')
 data['Title'] = data['Title'].replace(['Ms', 'Miss'], 'Miss')
 data['Title'] = data['Title'].replace(['Mlle', 'Mme','Lady', 'Countess', 'Dona'], 'Mrs')
 return data
train=titleExtract(train)
 test=titleExtract(test )<groupby>
for col in te.columns:
tr_rank = tr[col].rank(ascending=True)
 te_rank = te[col].rank(ascending=True)
 tr['{}_rank'.format(col)] = tr_rank.apply(lambda x:(x - tr_rank.min())/(tr_rank.max() - tr_rank.min()))
te['{}_rank'.format(col)] = te_rank.apply(lambda x:(x - te_rank.min())/(te_rank.max() - te_rank.min()))
print(train.groupby(['Title', 'Sex'])[['Survived']].mean())
 print(train.groupby(['Title', 'Sex'])[['Survived']].count() )<groupby>
def Stacking_trainer(tr, te, cols, clf, NAME, _scaler=False):
_tr, _te = tr.copy() , te.copy()
cv = StratifiedKFold(n_splits=5, random_state=46)
 X = _tr[cols].values
Y = _tr['target'].values
oof = np.zeros(len(tr))
preds = np.zeros(len(te))
_te = _te[cols]
if _scaler:
scaler = StandardScaler()
X = scaler.fit_transform(X)
 _te = scaler.transform(_te)
 for trn_index, val_index in cv.split(X, Y):
X_trn, Y_trn = X[trn_index], Y[trn_index]
X_val, Y_val = X[val_index], Y[val_index]
clf.fit(X_trn, Y_trn)
 oof[val_index] = clf.predict_proba(X_val)[:,1]
preds += clf.predict_proba(_te)[:,1] / cv.n_splits
tr[NAME] = oof
te[NAME] = preds
fig, axes = plt.subplots(nrows=1, ncols=2, figsize=(12, 4))
sns.distplot(tr[NAME], ax=axes[0])
 sns.distplot(te[NAME], ax=axes[1])
 axes[0].set_title('oof')
 axes[1].set_title('pred')
 plt.show()
print(NAME)
 print('AUC: {:.5f}'.format(roc_auc_score(tr['target'].values, oof)))
 return tr, te
print(train.groupby(['Title', 'Pclass'])[['Survived']].mean())
 print(train.groupby(['Title', 'Pclass'])[['Survived']].count() )<categorify>
tr_stage0 = tr.copy()
col_not_rank = [col for col in tr_stage0.columns if col not in ['target'] and 'rank' not in col]
col_in_rank = [col for col in tr_stage0.columns if col not in ['target'] and 'rank' in col]
def mapTitle(data):
title_map = {"Mr": 1, "Miss": 2, "Mrs": 3, "Master": 4}
data['Title'] = data['Title'].map(title_map ).astype(int)
 return data
train=mapTitle(train)
 test=mapTitle(test )<categorify>
%%time
stack_stage1_dict = {
'Stack_LR_1': LogisticRegression(penalty='l2', solver='liblinear', random_state=1),
'Stack_LR_2': LogisticRegression(penalty='l2', solver='liblinear', random_state=10),
'Stack_LR_3': LogisticRegression(penalty='l2', solver='liblinear', random_state=100),
'Stack_LR_4': LogisticRegression(penalty='l2', solver='liblinear', random_state=1000)
 }
for name, clf in stack_stage1_dict.items() :
tr, te = Stacking_trainer(tr, te, col_in_rank, clf, name, _scaler=False )
def fill_fare_nulls(data):
data.loc[(data.Fare.isnull())&(data.Pclass==1), 'Fare']=60.3
data.loc[(data.Fare.isnull())&(data.Pclass==2), 'Fare']=14.25
data.loc[(data.Fare.isnull())&(data.Pclass==3), 'Fare']=8.05
return data
test=fill_fare_nulls(test)
<feature_engineering>
tr['Stack_LR_Mean'] =(tr['Stack_LR_1'] + tr['Stack_LR_2'] + tr['Stack_LR_3'] + tr['Stack_LR_4'])/ 4
te['Stack_LR_Mean'] =(te['Stack_LR_1'] + te['Stack_LR_2'] + te['Stack_LR_3'] + te['Stack_LR_4'])/ 4
def splitClass_Train(data):
data.loc[(data.Fare>60.3)&(data.Pclass==1), 'PassengerCat']=0
data.loc[(data.Fare<=60.3)&(data.Pclass==1), 'PassengerCat']=1
data.loc[(data.Fare>14.3)&(data.Pclass==2), 'PassengerCat']=2
data.loc[(data.Fare<=14.3)&(data.Pclass==2), 'PassengerCat']=3
data.loc[(data.Fare>8.1)&(data.Pclass==3), 'PassengerCat']=4
data.loc[(data.Fare<=8.1)&(data.Pclass==3), 'PassengerCat']=5
data['PassengerCat']=data['PassengerCat'].astype(int)
 return data
train=splitClass_Train(train)
 def splitClass_Test(data):
data.loc[(data.Fare>60.00)&(data.Pclass==1), 'PassengerCat']=0
data.loc[(data.Fare<=60.00)&(data.Pclass==1), 'PassengerCat']=1
data.loc[(data.Fare>15.8)&(data.Pclass==2), 'PassengerCat']=2
data.loc[(data.Fare<=15.8)&(data.Pclass==2), 'PassengerCat']=3
data.loc[(data.Fare>7.9)&(data.Pclass==3), 'PassengerCat']=4
data.loc[(data.Fare<=7.9)&(data.Pclass==3), 'PassengerCat']=5
data['PassengerCat']=data['PassengerCat'].astype(int)
 return data
test=splitClass_Test(test)
<groupby>
def oof_distribution(tr, cols):
ncols = 3
nrows = int(np.ceil(len(cols)/ 3))
figsize =(12, 4 * nrows)
 fig, axes = plt.subplots(ncols=ncols, nrows=nrows, figsize=figsize)
 for col, ax in zip(cols, axes.ravel()):
auc = roc_auc_score(tr['target'], tr[col])
 tr_0 = tr[tr['target'] == 0]
tr_1 = tr[tr['target'] == 1]
tr_0 = tr_0[tr_0[col] > 0.0]
tr_0 = tr_0[tr_0[col] < 1.0]
tr_1 = tr_1[tr_1[col] > 0.0]
tr_1 = tr_1[tr_1[col] < 1.0]
tr_0 = tr_0.sample(5000)
 tr_1 = tr_1.sample(5000)
 sns.kdeplot(tr_0[col].values, ax=ax, shade=True, color="b", label='0')
 sns.kdeplot(tr_1[col].values, ax=ax, shade=True, color="r", label='1')
 ax.set_title('{} AUC: {:.5f}'.format(col, auc))
plt.tight_layout()
plt.show()
<groupby><EOS> 
submit = pd.read_csv('.. /input/sample_submission.csv')
 submit['target'] = te['Stack_LR_Mean'].values
submit.to_csv('submission.csv', index=False )