Kaggle competition: Instant Gratification
<SOS> metric: AUC Kaggle data source: instant-gratification<load_from_csv>
warnings.filterwarnings('ignore' )
train = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 test = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv')
 train.shape, test.shape<drop_column>
def load_data(data):
return pd.read_csv(data)
 with multiprocessing.Pool() as pool:
train, test, sub = pool.map(load_data, ['.. /input/train.csv', '.. /input/test.csv', '.. /input/sample_submission.csv'] )
train_ID = train['Id']
test_ID = test['Id']
train.drop(['Id'], axis=1, inplace=True)
 test.drop(['Id'], axis=1, inplace=True)
 train.shape, test.shape<feature_engineering>
cols = [c for c in train.columns if c not in ['id', 'target', 'wheezy-copper-turtle-magic']]
train["SalePrice"] = np.log1p(train["SalePrice"] )<drop_column>
oof_qda1 = np.zeros(len(train))
preds_qda1 = np.zeros(len(test))
train.drop(train[(train['OverallQual']<5)&(train['SalePrice']>200000)].index, inplace=True)
 train.drop(train[(train['GrLivArea']>4500)&(train['SalePrice']<300000)].index, inplace=True)
 train.reset_index(drop=True, inplace=True )<concatenate>
params = np.arange(0.1, 0.5, 0.1)
 parameters = [{'reg_param': params}]
reg_params = np.zeros(512 )
train_labels = train['SalePrice'].reset_index(drop=True)
 train_features = train.drop(['SalePrice'], axis=1)
 test_features = test
all_features = pd.concat([train_features, test_features] ).reset_index(drop=True)
 all_features.shape<groupby>
%%time
for i in tqdm_notebook(range(512)) :
train2 = train[train['wheezy-copper-turtle-magic']==i]
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train2.index
idx2 = test2.index
train2.reset_index(drop=True, inplace=True)
 test2.reset_index(drop=True, inplace=True)
 steps = [ 
('vt', VarianceThreshold(threshold=2)) , 
('sscaler', StandardScaler()),
]
pipe = Pipeline(steps=steps)
 data = pd.concat([pd.DataFrame(train2[cols]), pd.DataFrame(test2[cols])])
 data2 = pipe.fit_transform(data[cols])
 train3 = data2[:train2.shape[0]]
test3 = data2[train2.shape[0]:]
skf = StratifiedKFold(n_splits=11, random_state=42)
 for train_index, test_index in skf.split(train2, train2['target']):
qda = QuadraticDiscriminantAnalysis()
clf = GridSearchCV(qda, parameters, cv=4)
 clf.fit(train3[train_index,:], train2.loc[train_index]['target'])
 reg_params[i] = clf.best_params_['reg_param']
oof_qda1[idx1[test_index]] = clf.predict_proba(train3[test_index,:])[:,1]
preds_qda1[idx2] += clf.predict_proba(test3)[:,1] / skf.n_splits
def percent_missing(df):
data = pd.DataFrame(df)
 df_cols = list(pd.DataFrame(data))
dict_x = {}
for i in range(0, len(df_cols)) :
dict_x.update({df_cols[i]: round(data[df_cols[i]].isnull().mean() *100,2)})
 return dict_x
missing = percent_missing(all_features)
 df_miss = sorted(missing.items() , key=lambda x: x[1], reverse=True)
 print('Percent of missing data')
 df_miss[0:10]<data_type_conversions>
print('QDA with VarianceThreshold and StandardScaler CV Score =',round(roc_auc_score(train['target'], oof_qda1), 5))
all_features['MSSubClass'] = all_features['MSSubClass'].apply(str)
 all_features['YrSold'] = all_features['YrSold'].astype(str)
 all_features['MoSold'] = all_features['MoSold'].astype(str )<categorify>
sub['target'] = preds_qda1
sub['target'].to_csv('submission_qda1.csv', index=False )
def handle_missing(features):
features['Functional'] = features['Functional'].fillna('Typ')
 features['Electrical'] = features['Electrical'].fillna("SBrkr")
 features['KitchenQual'] = features['KitchenQual'].fillna("TA")
 features['Exterior1st'] = features['Exterior1st'].fillna(features['Exterior1st'].mode() [0])
 features['Exterior2nd'] = features['Exterior2nd'].fillna(features['Exterior2nd'].mode() [0])
 features['SaleType'] = features['SaleType'].fillna(features['SaleType'].mode() [0])
 features['MSZoning'] = features.groupby('MSSubClass')['MSZoning'].transform(lambda x: x.fillna(x.mode() [0]))
features["PoolQC"] = features["PoolQC"].fillna("None")
 for col in('GarageYrBlt', 'GarageArea', 'GarageCars'):
features[col] = features[col].fillna(0)
 for col in ['GarageType', 'GarageFinish', 'GarageQual', 'GarageCond']:
features[col] = features[col].fillna('None')
 for col in('BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2'):
features[col] = features[col].fillna('None')
 features['LotFrontage'] = features.groupby('Neighborhood')['LotFrontage'].transform(lambda x: x.fillna(x.median()))
objects = []
for i in features.columns:
if features[i].dtype == object:
objects.append(i)
 features.update(features[objects].fillna('None'))
numeric_dtypes = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
numeric = []
for i in features.columns:
if features[i].dtype in numeric_dtypes:
numeric.append(i)
 features.update(features[numeric].fillna(0))
return features
all_features = handle_missing(all_features )<sort_values>
sub['target'] = preds_qda1
sub['target'].hist(bins=100, alpha=0.6 )
missing = percent_missing(all_features)
 df_miss = sorted(missing.items() , key=lambda x: x[1], reverse=True)
 print('Percent of missing data')
 df_miss[0:10]<define_variables>
oof_qda2 = np.zeros(len(train))
preds_qda2 = np.zeros(len(test))
numeric_dtypes = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
numeric = []
for i in all_features.columns:
if all_features[i].dtype in numeric_dtypes:
numeric.append(i )<concatenate>
dict = dict()
for i in skew_index:
all_features[i] = boxcox1p(all_features[i], boxcox_normmax(all_features[i] + 1))<feature_engineering>
%%time
for s in range(512):
train2 = train[train['wheezy-copper-turtle-magic']==s]
test2 = test[test['wheezy-copper-turtle-magic']==s]
idx1 = train2.index
idx2 = test2.index
train2.reset_index(drop=True, inplace=True)
 sel = VarianceThreshold(threshold=1.5 ).fit(train2[cols])
 train3 = sel.transform(train2[cols])
 test3 = sel.transform(test2[cols])
 dict[s] = train3.shape[1]
all_features['BsmtFinType1_Unf'] = 1*(all_features['BsmtFinType1'] == 'Unf')
 all_features['HasWoodDeck'] =(all_features['WoodDeckSF'] == 0)* 1
all_features['HasOpenPorch'] =(all_features['OpenPorchSF'] == 0)* 1
all_features['HasEnclosedPorch'] =(all_features['EnclosedPorch'] == 0)* 1
all_features['Has3SsnPorch'] =(all_features['3SsnPorch'] == 0)* 1
all_features['HasScreenPorch'] =(all_features['ScreenPorch'] == 0)* 1
all_features['YearsSinceRemodel'] = all_features['YrSold'].astype(int)- all_features['YearRemodAdd'].astype(int)
 all_features['Total_Home_Quality'] = all_features['OverallQual'] + all_features['OverallCond']
all_features = all_features.drop(['Utilities', 'Street', 'PoolQC',], axis=1)
 all_features['TotalSF'] = all_features['TotalBsmtSF'] + all_features['1stFlrSF'] + all_features['2ndFlrSF']
all_features['YrBltAndRemod'] = all_features['YearBuilt'] + all_features['YearRemodAdd']
all_features['Total_sqr_footage'] =(all_features['BsmtFinSF1'] + all_features['BsmtFinSF2'] +
all_features['1stFlrSF'] + all_features['2ndFlrSF'])
 all_features['Total_Bathrooms'] =(all_features['FullBath'] +(0.5 * all_features['HalfBath'])+
all_features['BsmtFullBath'] +(0.5 * all_features['BsmtHalfBath']))
all_features['Total_porch_sf'] =(all_features['OpenPorchSF'] + all_features['3SsnPorch'] +
all_features['EnclosedPorch'] + all_features['ScreenPorch'] +
all_features['WoodDeckSF'])
 all_features['TotalBsmtSF'] = all_features['TotalBsmtSF'].apply(lambda x: np.exp(6)if x <= 0.0 else x)
 all_features['2ndFlrSF'] = all_features['2ndFlrSF'].apply(lambda x: np.exp(6.5)if x <= 0.0 else x)
 all_features['GarageArea'] = all_features['GarageArea'].apply(lambda x: np.exp(6)if x <= 0.0 else x)
 all_features['GarageCars'] = all_features['GarageCars'].apply(lambda x: 0 if x <= 0.0 else x)
 all_features['LotFrontage'] = all_features['LotFrontage'].apply(lambda x: np.exp(4.2)if x <= 0.0 else x)
 all_features['MasVnrArea'] = all_features['MasVnrArea'].apply(lambda x: np.exp(4)if x <= 0.0 else x)
 all_features['BsmtFinSF1'] = all_features['BsmtFinSF1'].apply(lambda x: np.exp(6.5)if x <= 0.0 else x)
 all_features['haspool'] = all_features['PoolArea'].apply(lambda x: 1 if x > 0 else 0)
 all_features['has2ndfloor'] = all_features['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)
 all_features['hasgarage'] = all_features['GarageArea'].apply(lambda x: 1 if x > 0 else 0)
 all_features['hasbsmt'] = all_features['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 all_features['hasfireplace'] = all_features['Fireplaces'].apply(lambda x: 1 if x > 0 else 0 )<feature_engineering>
%%time
for i in tqdm_notebook(range(512)) :
train2 = train[train['wheezy-copper-turtle-magic']==i]
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train2.index
idx2 = test2.index
train2.reset_index(drop=True, inplace=True)
 test2.reset_index(drop=True, inplace=True)
 steps = [ 
('scaler', StandardScaler()), 
('fa', FeatureAgglomeration(n_clusters=dict[i]))
]
pipe = Pipeline(steps=steps)
 data = pd.concat([pd.DataFrame(train2[cols]), pd.DataFrame(test2[cols])])
 data2 = pipe.fit_transform(data[cols])
 train3 = data2[:train2.shape[0]]
test3 = data2[train2.shape[0]:]
skf = StratifiedKFold(n_splits=11, random_state=42)
 for train_index, test_index in skf.split(train2, train2['target']):
clf = QuadraticDiscriminantAnalysis(0.5)
 clf.fit(train3[train_index,:], train2.loc[train_index]['target'])
 oof_qda2[idx1[test_index]] = clf.predict_proba(train3[test_index,:])[:,1]
preds_qda2[idx2] += clf.predict_proba(test3)[:,1] / skf.n_splits
def logs(res, ls):
m = res.shape[1]
for l in ls:
res = res.assign(newcol=pd.Series(np.log(1.01+res[l])).values)
 res.columns.values[m] = l + '_log'
m += 1
return res
log_features = ['LotFrontage','LotArea','MasVnrArea','BsmtFinSF1','BsmtFinSF2','BsmtUnfSF',
'TotalBsmtSF','1stFlrSF','2ndFlrSF','LowQualFinSF','GrLivArea',
'BsmtFullBath','BsmtHalfBath','FullBath','HalfBath','BedroomAbvGr','KitchenAbvGr',
'TotRmsAbvGrd','Fireplaces','GarageCars','GarageArea','WoodDeckSF','OpenPorchSF',
'EnclosedPorch','3SsnPorch','ScreenPorch','PoolArea','MiscVal','YearRemodAdd','TotalSF']
all_features = logs(all_features, log_features )<compute_test_metric>
print('QDA with PCA CV Score =',round(roc_auc_score(train['target'], oof_qda2), 5))
def squares(res, ls):
m = res.shape[1]
for l in ls:
res = res.assign(newcol=pd.Series(res[l]*res[l] ).values)
 res.columns.values[m] = l + '_sq'
m += 1
return res
squared_features = ['YearRemodAdd', 'LotFrontage_log',
'TotalBsmtSF_log', '1stFlrSF_log', '2ndFlrSF_log', 'GrLivArea_log',
'GarageCars_log', 'GarageArea_log']
all_features = squares(all_features, squared_features )<categorify>
sub['target'] = preds_qda2
sub.to_csv('submission_qda2.csv', index=False )
all_features = pd.get_dummies(all_features ).reset_index(drop=True)
 all_features.shape<count_duplicates>
sub['target'] = preds_qda2
sub['target'].hist(bins=100, alpha=0.6 )
all_features = all_features.loc[:,~all_features.columns.duplicated() ]<choose_model_class>
oof_gmm = np.zeros(len(train))
preds_gmm = np.zeros(len(test))
kf = KFold(n_splits=12, random_state=42, shuffle=True )<compute_test_metric>
def get_mean_cov(x, y):
model = GraphicalLasso()
ones =(y==1 ).astype(bool)
 x2 = x[ones]
model.fit(x2)
 p1 = model.precision_
m1 = model.location_
onesb =(y==0 ).astype(bool)
 x2b = x[onesb]
model.fit(x2b)
 p2 = model.precision_
m2 = model.location_
ms = np.stack([m1,m2])
 ps = np.stack([p1,p2])
 return ms, ps
def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))
def cv_rmse(model, X=X):
rmse = np.sqrt(-cross_val_score(model, X, train_labels, scoring="neg_mean_squared_error", cv=kf))
return(rmse )<choose_model_class>
%%time
for i in tqdm_notebook(range(512)) :
train2 = train[train['wheezy-copper-turtle-magic']==i]
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train2.index
idx2 = test2.index
train2.reset_index(drop=True, inplace=True)
 test2.reset_index(drop=True, inplace=True)
 steps = [ 
('vt', VarianceThreshold(threshold=1.5))
]
pipe = Pipeline(steps=steps)
 data = pd.concat([pd.DataFrame(train2[cols]), pd.DataFrame(test2[cols])])
 data2 = pipe.fit_transform(data[cols])
 train3 = data2[:train2.shape[0]]
test3 = data2[train2.shape[0]:]
skf = StratifiedKFold(n_splits=11, random_state=42, shuffle=True)
 for train_index, test_index in skf.split(train3, train2['target']):
ms, ps = get_mean_cov(train3[train_index,:], train2.loc[train_index]['target'].values)
 clf = GaussianMixture(n_components=2, init_params='random', covariance_type='full', tol=0.001, reg_covar=0.001, max_iter=100, n_init=1, means_init=ms, precisions_init=ps)
 clf.fit(np.concatenate([train3,test3],axis = 0))
oof_gmm[idx1[test_index]] = clf.predict_proba(train3[test_index,:])[:,0]
preds_gmm[idx2] += clf.predict_proba(test3)[:,0] / skf.n_splits
lightgbm = LGBMRegressor(objective='regression',
num_leaves=6,
learning_rate=0.01,
n_estimators=7000,
max_bin=200,
bagging_fraction=0.8,
bagging_freq=4,
bagging_seed=8,
feature_fraction=0.2,
feature_fraction_seed=8,
min_sum_hessian_in_leaf = 11,
verbose=-1,
random_state=42)
 xgboost = XGBRegressor(learning_rate=0.01,
n_estimators=6000,
max_depth=4,
min_child_weight=0,
gamma=0.6,
subsample=0.7,
colsample_bytree=0.7,
objective='reg:linear',
nthread=-1,
scale_pos_weight=1,
seed=27,
reg_alpha=0.00006,
random_state=42)
 ridge_alphas = [1e-15, 1e-10, 1e-8, 9e-4, 7e-4, 5e-4, 3e-4, 1e-4, 1e-3, 5e-2, 1e-2, 0.1, 0.3, 1, 3, 5, 10, 15, 18, 20, 30, 50, 75, 100]
ridge = make_pipeline(RobustScaler() , RidgeCV(alphas=ridge_alphas, cv=kf))
svr = make_pipeline(RobustScaler() , SVR(C= 20, epsilon= 0.008, gamma=0.0003))
gbr = GradientBoostingRegressor(n_estimators=6000,
learning_rate=0.01,
max_depth=4,
max_features='sqrt',
min_samples_leaf=15,
min_samples_split=10,
loss='huber',
random_state=42)
 rf = RandomForestRegressor(n_estimators=1200,
max_depth=15,
min_samples_split=5,
min_samples_leaf=5,
max_features=None,
oob_score=True,
random_state=42)
 stack_gen = StackingCVRegressor(regressors=(xgboost, lightgbm, svr, ridge, gbr, rf),
meta_regressor=xgboost,
use_features_in_secondary=True )<compute_test_metric>
print('GaussianMixture with VarianceThreshold CV Score =',round(roc_auc_score(train['target'], oof_gmm), 5))
scores = {}
score = cv_rmse(lightgbm)
 print("lightgbm: {:.4f}({:.4f})".format(score.mean() , score.std()))
scores['lgb'] =(score.mean() , score.std() )<compute_test_metric>
sub['target'] = preds_gmm
sub.to_csv('submission_gmm.csv', index=False )
score = cv_rmse(xgboost)
 print("xgboost: {:.4f}({:.4f})".format(score.mean() , score.std()))
scores['xgb'] =(score.mean() , score.std() )<compute_test_metric>
oof_lr = np.zeros(len(train))
preds_lr = np.zeros(len(test))
score = cv_rmse(svr)
 print("SVR: {:.4f}({:.4f})".format(score.mean() , score.std()))
scores['svr'] =(score.mean() , score.std() )<compute_test_metric>
%%time
for i in tqdm_notebook(range(512)) :
train2 = train[train['wheezy-copper-turtle-magic']==i]
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train2.index
idx2 = test2.index
train2.reset_index(drop=True, inplace=True)
 test2.reset_index(drop=True, inplace=True)
 steps = [ 
('vt', VarianceThreshold(threshold=1.5)) , 
('poly', PolynomialFeatures(degree=2)) , 
('sc', StandardScaler())
 ]
pipe = Pipeline(steps=steps)
 data = pd.concat([pd.DataFrame(train2[cols]), pd.DataFrame(test2[cols])])
 data2 = pipe.fit_transform(data[cols])
 train3 = data2[:train2.shape[0]]
test3 = data2[train2.shape[0]:]
skf = StratifiedKFold(n_splits=11, random_state=42, shuffle=True)
 for train_index, test_index in skf.split(train3, train2['target']):
clf = LogisticRegression(solver='saga', penalty='l2', C=0.01, tol=0.001, n_jobs=-1)
 clf.fit(train3[train_index,:], train2.loc[train_index]['target'])
 oof_lr[idx1[test_index]] = clf.predict_proba(train3[test_index,:])[:,1]
preds_lr[idx2] += clf.predict_proba(test3)[:,1] / skf.n_splits
score = cv_rmse(ridge)
 print("ridge: {:.4f}({:.4f})".format(score.mean() , score.std()))
scores['ridge'] =(score.mean() , score.std() )<compute_test_metric>
print('Logistic Regression with VarianceThreshold CV Score =',round(roc_auc_score(train['target'], oof_lr), 5))
score = cv_rmse(rf)
 print("rf: {:.4f}({:.4f})".format(score.mean() , score.std()))
scores['rf'] =(score.mean() , score.std() )<compute_test_metric>
sub['target'] = preds_lr
sub.to_csv('submission_lr.csv', index=False )
score = cv_rmse(gbr)
 print("gbr: {:.4f}({:.4f})".format(score.mean() , score.std()))
scores['gbr'] =(score.mean() , score.std() )<train_model>
oof_ls = np.zeros(len(train))
preds_ls = np.zeros(len(test))
print('stack_gen')
 stack_gen_model = stack_gen.fit(np.array(X), np.array(train_labels))<train_model>
%%time
for i in tqdm_notebook(range(512)) :
train2 = train[train['wheezy-copper-turtle-magic']==i]
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train2.index
idx2 = test2.index
train2.reset_index(drop=True, inplace=True)
 test2.reset_index(drop=True, inplace=True)
 steps = [ 
('vt', VarianceThreshold(threshold=1.5))
]
pipe = Pipeline(steps=steps)
 data = pd.concat([pd.DataFrame(train2[cols]), pd.DataFrame(test2[cols])])
 data2 = pipe.fit_transform(data[cols])
 train3 = data2[:train2.shape[0]]
test3 = data2[train2.shape[0]:]
skf = StratifiedKFold(n_splits=11, random_state=42, shuffle=True)
 for train_index, test_index in skf.split(train3, train2['target']):
clf = LabelSpreading(gamma=0.01, kernel='rbf', max_iter=10)
 clf.fit(train3[train_index,:], train2.loc[train_index]['target'])
 oof_ls[idx1[test_index]] = clf.predict_proba(train3[test_index,:])[:,1]
preds_ls[idx2] += clf.predict_proba(test3)[:,1] / skf.n_splits
print('lightgbm')
 lgb_model_full_data = lightgbm.fit(X, train_labels )<train_model>
print('LabelSpreading with VarianceThreshold CV Score =',round(roc_auc_score(train['target'], oof_ls), 5))
print('xgboost')
 xgb_model_full_data = xgboost.fit(X, train_labels )<train_model>
sub['target'] = preds_ls
sub.to_csv('submission_ls.csv', index=False )
print('Svr')
 svr_model_full_data = svr.fit(X, train_labels )<train_model>
oof_knn = np.zeros(len(train))
preds_knn = np.zeros(len(test))
print('Ridge')
 ridge_model_full_data = ridge.fit(X, train_labels )<train_model>
%%time
for i in tqdm_notebook(range(512)) :
train2 = train[train['wheezy-copper-turtle-magic']==i]
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train2.index
idx2 = test2.index
train2.reset_index(drop=True, inplace=True)
 test.reset_index(drop=True, inplace=True)
 steps = [ 
('vt', VarianceThreshold(threshold=2)) , 
('scaler', StandardScaler())
 ]
pipe = Pipeline(steps=steps)
 data = pd.concat([pd.DataFrame(train2[cols]), pd.DataFrame(test2[cols])])
 data2 = pipe.fit_transform(data[cols])
 train3 = data2[:train2.shape[0]]
test3 = data2[train2.shape[0]:]
skf = StratifiedKFold(n_splits=11, random_state=42)
 for train_index, test_index in skf.split(train2, train2['target']):
clf = KNeighborsClassifier(n_neighbors=17, p=2.9, n_jobs=-1)
 clf.fit(train3[train_index,:], train2.loc[train_index]['target'])
 oof_knn[idx1[test_index]] = clf.predict_proba(train3[test_index,:])[:,1]
preds_knn[idx2] += clf.predict_proba(test3)[:,1] / skf.n_splits
print('RandomForest')
 rf_model_full_data = rf.fit(X, train_labels )<train_model>
print('kNN with VarianceThreshold and StandardScaler CV Score =',round(roc_auc_score(train['target'], oof_knn), 5))
print('GradientBoosting')
 gbr_model_full_data = gbr.fit(X, train_labels )<predict_on_test>
sub['target'] = preds_knn
sub.to_csv('submission_knn.csv', index=False )
def blended_predictions(X):
return(( 0.1 * ridge_model_full_data.predict(X)) + \ 
(0.2 * svr_model_full_data.predict(X)) + \ 
(0.1 * gbr_model_full_data.predict(X)) + \ 
(0.1 * xgb_model_full_data.predict(X)) + \ 
(0.1 * lgb_model_full_data.predict(X)) + \ 
(0.05 * rf_model_full_data.predict(X)) + \ 
(0.35 * stack_gen_model.predict(np.array(X))))<compute_test_metric>
sub['target'] = preds_knn
sub['target'].hist(bins=100, alpha=0.6 )
blended_score = rmsle(train_labels, blended_predictions(X))
scores['blended'] =(blended_score, 0)
 print('RMSLE score on train data:')
 print(blended_score )<load_from_csv>
oof_nn = np.zeros(len(train))
preds_nn = np.zeros(len(test))
submission = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/sample_submission.csv")
 submission.shape<predict_on_test>
%%time
for i in tqdm_notebook(range(512)) :
train2 = train[train['wheezy-copper-turtle-magic']==i]
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train2.index
idx2 = test2.index
train2.reset_index(drop=True, inplace=True)
 test2.reset_index(drop=True, inplace=True)
 steps = [ 
('vt', VarianceThreshold(threshold=1.5)) , 
('poly', PolynomialFeatures(degree=2)) , 
('sc', StandardScaler())
 ]
pipe = Pipeline(steps=steps)
 data = pd.concat([pd.DataFrame(train2[cols]), pd.DataFrame(test2[cols])])
 data2 = pipe.fit_transform(data[cols])
 train3 = data2[:train2.shape[0]]
test3 = data2[train2.shape[0]:]
skf = StratifiedKFold(n_splits=11, random_state=42, shuffle=True)
 for train_index, test_index in skf.split(train3, train2['target']):
clf = MLPClassifier(random_state=3, activation='relu', solver='lbfgs', tol=1e-06, hidden_layer_sizes=(250,))
clf.fit(train3[train_index,:], train2.loc[train_index]['target'])
 oof_nn[idx1[test_index]] = clf.predict_proba(train3[test_index,:])[:,1]
preds_nn[idx2] += clf.predict_proba(test3)[:,1] / skf.n_splits
submission.iloc[:,1] = np.floor(np.expm1(blended_predictions(X_test)) )<save_to_csv>
print('NN with PolynomialFeatures and StandardScaler CV Score =',round(roc_auc_score(train['target'], oof_nn), 5))
q1 = submission['SalePrice'].quantile(0.0045)
 q2 = submission['SalePrice'].quantile(0.99)
 submission['SalePrice'] = submission['SalePrice'].apply(lambda x: x if x > q1 else x*0.77)
 submission['SalePrice'] = submission['SalePrice'].apply(lambda x: x if x < q2 else x*1.1)
 submission.to_csv("submission_regression1.csv", index=False )<save_to_csv>
sub['target'] = preds_nn
sub.to_csv('submission_nn.csv', index=False )
submission['SalePrice'] *= 1.001619
submission.to_csv("submission_regression2.csv", index=False )<set_options>
oof_qda3 = np.zeros(len(train))
preds_qda3 = preds_gmm.copy()
warnings.filterwarnings('ignore' )<load_from_csv>
%%time
for i in range(10):
test['target'] = preds_qda3
test.loc[test['target'] > 0.955, 'target'] = 1
test.loc[test['target'] < 0.045, 'target'] = 0
usefull_test = test[(test['target'] == 1)|(test['target'] == 0)]
new_train = pd.concat([train, usefull_test] ).reset_index(drop=True)
 print(usefull_test.shape[0], "Test Record added for iteration : ", i + 1)
 new_train.loc[oof_qda3 > 0.995, 'target'] = 1
new_train.loc[oof_qda3 < 0.005, 'target'] = 0
oof_qda3 = np.zeros(len(train))
preds_qda = np.zeros(len(test))
for i in tqdm_notebook(range(512)) :
train2 = new_train[new_train['wheezy-copper-turtle-magic']==i]
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train[train['wheezy-copper-turtle-magic']==i].index
idx2 = test2.index
train2.reset_index(drop=True, inplace=True)
 test2.reset_index(drop=True, inplace=True)
 steps = [ 
('vt', VarianceThreshold(threshold=1.5)) , 
('scaler', StandardScaler())
 ]
pipe = Pipeline(steps=steps)
 train3 = pipe.fit_transform(train2[cols])
 test3 = pipe.fit_transform(test2[cols])
 skf = StratifiedKFold(n_splits=11, random_state=42)
 for train_index, test_index in skf.split(train2, train2['target']):
oof_test_index = [t for t in test_index if t < len(idx1)]
clf = QuadraticDiscriminantAnalysis(reg_params[i])
 clf.fit(train3[train_index,:],train2.loc[train_index]['target'])
 if len(oof_test_index)> 0:
oof_qda3[idx1[oof_test_index]] = clf.predict_proba(train3[oof_test_index,:])[:,1]
preds_qda3[idx2] += clf.predict_proba(test3)[:,1] / skf.n_splits
print('QDA with Pseued Label by GaussianMixture CV Score =',round(roc_auc_score(train['target'], oof_qda3), 5))
train_df = pd.read_csv('.. /input/titanic/train.csv')
 test_df = pd.read_csv('.. /input/titanic/test.csv')
 df = [train_df, test_df]<count_missing_values>
sub['target'] = preds_qda3
sub.to_csv('submission_gda3.csv', index=False )
train_df.isnull().sum()<count_missing_values>
oof_qda4 = oof_qda1.copy()
preds_qda4 = preds_qda1.copy()
test_df.isnull().sum()<count_values>
%%time
for i in range(10):
test['target'] = preds_qda4
test.loc[test['target'] > 0.955, 'target'] = 1
test.loc[test['target'] < 0.045, 'target'] = 0
usefull_test = test[(test['target'] == 1)|(test['target'] == 0)]
new_train = pd.concat([train, usefull_test] ).reset_index(drop=True)
 print(usefull_test.shape[0], "Test Record added for iteration : ", i + 1)
 new_train.loc[oof_qda4 > 0.995, 'target'] = 1
new_train.loc[oof_qda4 < 0.005, 'target'] = 0
oof_qda4= np.zeros(len(train))
preds_qd4 = np.zeros(len(test))
for i in tqdm_notebook(range(512)) :
train2 = new_train[new_train['wheezy-copper-turtle-magic']==i]
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train[train['wheezy-copper-turtle-magic']==i].index
idx2 = test2.index
train2.reset_index(drop=True, inplace=True)
 test2.reset_index(drop=True, inplace=True)
 steps = [ 
('vt', VarianceThreshold(threshold=1.5)) , 
('scaler', StandardScaler())
 ]
pipe = Pipeline(steps=steps)
 train3 = pipe.fit_transform(train2[cols])
 test3 = pipe.fit_transform(test2[cols])
 skf = StratifiedKFold(n_splits=11, random_state=42)
 for train_index, test_index in skf.split(train2, train2['target']):
oof_test_index = [t for t in test_index if t < len(idx1)]
clf = QuadraticDiscriminantAnalysis(reg_params[i])
 clf.fit(train3[train_index,:],train2.loc[train_index]['target'])
 if len(oof_test_index)> 0:
oof_qda4[idx1[oof_test_index]] = clf.predict_proba(train3[oof_test_index,:])[:,1]
preds_qda4[idx2] += clf.predict_proba(test3)[:,1] / skf.n_splits
print('GSearch QDA with Pseued Label by QDA CV Score =',round(roc_auc_score(train['target'], oof_qda3), 5))
train_df.Embarked.value_counts()<data_type_conversions>
sub['target'] = preds_qda4
sub.to_csv('submission_gda3.csv', index=False )
train_df['Embarked'] = train_df['Embarked'].fillna(train_df['Embarked'].mode() [0])
 df = [train_df, test_df]<groupby>
sub['target'] = preds_qda4
sub['target'].hist(bins=100, alpha=0.6 )
train_df[['Embarked', 'Survived']].groupby(['Embarked'], as_index=False ).mean()<data_type_conversions>
print('All Model CV Score Summary')
 print('---------------------------------')
 print('1.QDA with VarianceThreshold and StandardScaler', roc_auc_score(train['target'], oof_qda1))
print('2.QDA with PCA', roc_auc_score(train['target'], oof_qda2))
print('3.GaussianMixture with VarianceThreshold', roc_auc_score(train['target'], oof_gmm))
print('4.Logistic Regression with PolynomialFeatures and StandardScaler', roc_auc_score(train['target'], oof_lr))
print('5.LabelSpreading with VarianceThreshold', roc_auc_score(train['target'], oof_ls))
print('6.kNN with VarianceThreshold and StandardScaler', roc_auc_score(train['target'], oof_knn))
print('P1.QDA with Pseued Label by GaussianMixture', roc_auc_score(train['target'], oof_qda3))
for dataset in df:
dataset['Embarked'] = dataset['Embarked'].map({'S': 0, 'C': 1, 'Q': 2} ).astype(int)
 train_df.head()<count_values>
preds_list = [preds_qda1, preds_qda2, preds_gmm, preds_lr, preds_ls, preds_knn, preds_nn, preds_qda3]
oof_list = [oof_qda1, oof_qda2, oof_gmm, oof_lr, oof_ls, oof_knn, oof_nn, oof_qda3]
print('The number of model is {}'.format(len(preds_list)) )
train_df.Parch.value_counts()<count_values>
oof_avg = sum(oof_list)/ len(oof_list)
 preds_avg = sum(preds_list)/ len(preds_list )
train_df.SibSp.value_counts()<sort_values>
print('{} model blend CV score ='.format(len(preds_list)) ,round(roc_auc_score(train['target'], oof_avg),6))
train_df[['Parch', 'Survived']].groupby(['Parch'], as_index=False ).mean().sort_values(by='Survived', ascending=False )<sort_values>
sub['target'] = preds_avg
sub.to_csv('submission_blend_avg.csv', index=False )
train_df[['SibSp', 'Survived']].groupby(['SibSp'], as_index=False ).mean().sort_values(by='Survived', ascending=False )<sort_values>
oof_blend = np.zeros(len(train))
preds_knn = np.zeros(len(test))
for dataset in df:
dataset['FamilySize'] = dataset['SibSp'] + dataset['Parch'] + 1
train_df[['FamilySize', 'Survived']].groupby(['FamilySize'], as_index=False ).mean().sort_values(by='Survived', ascending=False )<sort_values>
oof_blend = 0.5*(oof_qda3 + oof_qda4)
 preds_blend = 0.5*(preds_qda3 + preds_qda4 )
for dataset in df:
dataset['IsAlone'] = 0
dataset.loc[dataset['FamilySize'] == 1, 'IsAlone'] = 1
train_df[['IsAlone', 'Survived']].groupby(['IsAlone'], as_index=False ).mean().sort_values(by='Survived', ascending=False )<count_values>
print('Stacking model blend CV score =',round(roc_auc_score(train['target'], oof_blend),6))
train_df.Sex.value_counts()<groupby>
sub['target'] = preds_blend
sub.to_csv('submission_blend.csv', index=False )
train_df[['Sex', 'Survived']].groupby(['Sex'], as_index=False ).mean()<data_type_conversions>
sub['target'] = preds_blend
sub['target'].hist(bins=100, alpha=0.6 )
for dataset in df:
dataset['Sex'] = dataset['Sex'].map({'female': 1, 'male': 0} ).astype(int )<feature_engineering>
oof_qda1 = oof_qda1.reshape(-1, 1)
 oof_qda2 = oof_qda2.reshape(-1, 1)
 oof_gmm = oof_gmm.reshape(-1, 1)
 oof_lr = oof_lr.reshape(-1, 1)
 oof_ls = oof_ls.reshape(-1, 1)
 oof_knn = oof_knn.reshape(-1, 1)
 oof_nn = oof_nn.reshape(-1, 1)
 oof_qda3 = oof_qda3.reshape(-1, 1 )
for dataset in df:
dataset['Ticket_num'] = dataset['Ticket'].apply(lambda x: func(x))
dataset['Ticket_num'] = dataset['Ticket_num'].replace('LINE', 0 )<count_unique_values>
preds_qda1 = preds_qda1.reshape(-1, 1)
 preds_qda2 = preds_qda2.reshape(-1, 1)
 preds_gmm = preds_gmm.reshape(-1, 1)
 preds_lr = preds_lr.reshape(-1, 1)
 preds_ls = preds_ls.reshape(-1, 1)
 preds_knn = preds_knn.reshape(-1, 1)
 preds_nn = preds_nn.reshape(-1, 1)
 preds_qda3 = preds_qda3.reshape(-1, 1 )
train_df.Ticket_num.unique().shape<feature_engineering>
train_stck = np.concatenate([oof_qda1, oof_qda2, oof_gmm, oof_lr, oof_ls, oof_knn, oof_nn, oof_qda3], axis=1)
 test_stack = np.concatenate([preds_qda1, preds_qda2, preds_gmm, preds_lr, preds_ls, preds_knn, preds_nn, preds_qda3], axis=1 )
for dataset in df:
dataset['Title'] = dataset['Name'].str.extract('([A-Za-z]+)\.', expand=False)
 pd.crosstab(train_df['Title'], train_df['Sex'] )<groupby>
oof_stack = np.zeros(len(train))
pred_stack = np.zeros(len(test))
train_df[['Title', 'Age']].groupby(['Title'], as_index=False ).mean()<feature_engineering>
for train_index, test_index in skf.split(train_stck, train['target']):
clf = LogisticRegression(solver='saga', penalty='l2', C=0.01, tol=0.001)
 clf.fit(train_stck[train_index], train['target'][train_index])
 oof_stack[test_index] = clf.predict_proba(train_stck[test_index,:])[:,1]
pred_stack += clf.predict_proba(test_stack)[:,1] / skf.n_splits
for dataset in df:
dataset['Age_fill'] = 0
dataset.loc[dataset['Title'] == 'Capt', 'Age_fill'] = 70
dataset.loc[dataset['Title'] == 'Col', 'Age_fill'] = 58
dataset.loc[dataset['Title'] == 'Countess', 'Age_fill'] = 33
dataset.loc[dataset['Title'] == 'Don', 'Age_fill'] = 40
dataset.loc[dataset['Title'] == 'Dr', 'Age_fill'] = 42
dataset.loc[dataset['Title'] == 'jonkheer', 'Age_fill'] = 38
dataset.loc[dataset['Title'] == 'Lady', 'Age_fill'] = 48
dataset.loc[dataset['Title'] == 'Major', 'Age_fill'] = 48.5
dataset.loc[dataset['Title'] == 'Master', 'Age_fill'] = 4.6
dataset.loc[dataset['Title'] == 'Miss', 'Age_fill'] = 21.8
dataset.loc[dataset['Title'] == 'Mlle', 'Age_fill'] = 24
dataset.loc[dataset['Title'] == 'Mme', 'Age_fill'] = 24
dataset.loc[dataset['Title'] == 'Mr', 'Age_fill'] = 32.4
dataset.loc[dataset['Title'] == 'Mrs', 'Age_fill'] = 35.9
dataset.loc[dataset['Title'] == 'Ms', 'Age_fill'] = 28
dataset.loc[dataset['Title'] == 'Rev', 'Age_fill'] = 43.2
dataset.loc[dataset['Title'] == 'Sir', 'Age_fill'] = 49<data_type_conversions>
print('{} model stack CV score ='.format(len(preds_list)) ,round(roc_auc_score(train['target'], oof_stack),6))
<count_missing_values><EOS> 
sub['target'] = pred_stack
sub.to_csv('submission_stack.csv', index=False )