Kaggle competition: House Prices - Advanced Regression Techniques
%%time
for i in range(10):
test['target'] = preds_qda3
test.loc[test['target'] > 0.955, 'target'] = 1
test.loc[test['target'] < 0.045, 'target'] = 0
usefull_test = test[(test['target'] == 1)|(test['target'] == 0)]
new_train = pd.concat([train, usefull_test] ).reset_index(drop=True)
 print(usefull_test.shape[0], "Test Record added for iteration : ", i + 1)
 new_train.loc[oof_qda3 > 0.995, 'target'] = 1
new_train.loc[oof_qda3 < 0.005, 'target'] = 0
oof_qda3 = np.zeros(len(train))
preds_qda = np.zeros(len(test))
for i in tqdm_notebook(range(512)) :
train2 = new_train[new_train['wheezy-copper-turtle-magic']==i]
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train[train['wheezy-copper-turtle-magic']==i].index
idx2 = test2.index
train2.reset_index(drop=True, inplace=True)
 test2.reset_index(drop=True, inplace=True)
 steps = [ 
('vt', VarianceThreshold(threshold=1.5)) , 
('scaler', StandardScaler())
 ]
pipe = Pipeline(steps=steps)
 train3 = pipe.fit_transform(train2[cols])
 test3 = pipe.fit_transform(test2[cols])
 skf = StratifiedKFold(n_splits=11, random_state=42)
 for train_index, test_index in skf.split(train2, train2['target']):
oof_test_index = [t for t in test_index if t < len(idx1)]
clf = QuadraticDiscriminantAnalysis(reg_params[i])
 clf.fit(train3[train_index,:],train2.loc[train_index]['target'])
 if len(oof_test_index)> 0:
oof_qda3[idx1[oof_test_index]] = clf.predict_proba(train3[oof_test_index,:])[:,1]
preds_qda3[idx2] += clf.predict_proba(test3)[:,1] / skf.n_splits
print('QDA with Pseued Label by GaussianMixture CV Score =',round(roc_auc_score(train['target'], oof_qda3), 5))<save_to_csv>
train = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/train.csv")
 train.head()
sub['target'] = preds_qda3
sub.to_csv('submission_gda3.csv', index=False )<create_dataframe>
test = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/test.csv")
 TEST_ID = test['Id']
test.head()
oof_qda4 = oof_qda1.copy()
preds_qda4 = preds_qda1.copy()<feature_engineering>
( train.corr() **2)["SalePrice"].sort_values(ascending = False)[1:]
train.corr() ['SalePrice'].sort_values(ascending = False)
%%time
for i in range(10):
test['target'] = preds_qda4
test.loc[test['target'] > 0.955, 'target'] = 1
test.loc[test['target'] < 0.045, 'target'] = 0
usefull_test = test[(test['target'] == 1)|(test['target'] == 0)]
new_train = pd.concat([train, usefull_test] ).reset_index(drop=True)
 print(usefull_test.shape[0], "Test Record added for iteration : ", i + 1)
 new_train.loc[oof_qda4 > 0.995, 'target'] = 1
new_train.loc[oof_qda4 < 0.005, 'target'] = 0
oof_qda4= np.zeros(len(train))
preds_qd4 = np.zeros(len(test))
for i in tqdm_notebook(range(512)) :
train2 = new_train[new_train['wheezy-copper-turtle-magic']==i]
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train[train['wheezy-copper-turtle-magic']==i].index
idx2 = test2.index
train2.reset_index(drop=True, inplace=True)
 test2.reset_index(drop=True, inplace=True)
 steps = [ 
('vt', VarianceThreshold(threshold=1.5)) , 
('scaler', StandardScaler())
 ]
pipe = Pipeline(steps=steps)
 train3 = pipe.fit_transform(train2[cols])
 test3 = pipe.fit_transform(test2[cols])
 skf = StratifiedKFold(n_splits=11, random_state=42)
 for train_index, test_index in skf.split(train2, train2['target']):
oof_test_index = [t for t in test_index if t < len(idx1)]
clf = QuadraticDiscriminantAnalysis(reg_params[i])
 clf.fit(train3[train_index,:],train2.loc[train_index]['target'])
 if len(oof_test_index)> 0:
oof_qda4[idx1[oof_test_index]] = clf.predict_proba(train3[oof_test_index,:])[:,1]
preds_qda4[idx2] += clf.predict_proba(test3)[:,1] / skf.n_splits
print('GSearch QDA with Pseued Label by QDA CV Score =',round(roc_auc_score(train['target'], oof_qda3), 5))<save_to_csv>
train.loc[train.GrLivArea > 4000].loc[train.SalePrice <= 200000]
sub['target'] = preds_qda4
sub.to_csv('submission_gda3.csv', index=False )<prepare_output>
train.loc[train.TotalBsmtSF > 6000]
sub['target'] = preds_qda4
sub['target'].hist(bins=100, alpha=0.6 )<compute_train_metric>
train.loc[train['1stFlrSF'] > 4000]
print('All Model CV Score Summary')
 print('---------------------------------')
 print('1.QDA with VarianceThreshold and StandardScaler', roc_auc_score(train['target'], oof_qda1))
print('2.QDA with PCA', roc_auc_score(train['target'], oof_qda2))
print('3.GaussianMixture with VarianceThreshold', roc_auc_score(train['target'], oof_gmm))
print('4.Logistic Regression with PolynomialFeatures and StandardScaler', roc_auc_score(train['target'], oof_lr))
print('5.LabelSpreading with VarianceThreshold', roc_auc_score(train['target'], oof_ls))
print('6.kNN with VarianceThreshold and StandardScaler', roc_auc_score(train['target'], oof_knn))
print('P1.QDA with Pseued Label by GaussianMixture', roc_auc_score(train['target'], oof_qda3)) 
<define_variables>
train.loc[1990 < train.YearRemodAdd].loc[train.YearRemodAdd < 2000].loc[train.SalePrice > 600000]
preds_list = [preds_qda1, preds_qda2, preds_gmm, preds_lr, preds_ls, preds_knn, preds_nn, preds_qda3]
oof_list = [oof_qda1, oof_qda2, oof_gmm, oof_lr, oof_ls, oof_knn, oof_nn, oof_qda3]
print('The number of model is {}'.format(len(preds_list)) )<groupby>
train.drop(train.index[[523, 691, 1169, 1182, 1298]] )
oof_avg = sum(oof_list)/ len(oof_list)
 preds_avg = sum(preds_list)/ len(preds_list )<compute_test_metric>
all_data = pd.concat(( train, test)).reset_index(drop = True )
print('{} model blend CV score ='.format(len(preds_list)) ,round(roc_auc_score(train['target'], oof_avg),6))<save_to_csv>
for feature in all_data.select_dtypes(exclude=['object'] ).columns:
all_data[feature].fillna(all_data[feature].median() , inplace = True)
 for feature in all_data.select_dtypes(include=['object'] ).columns:
all_data[feature].fillna(all_data[feature].value_counts().idxmax() , inplace = True )
sub['target'] = preds_avg
sub.to_csv('submission_blend_avg.csv', index=False )<define_variables>
numeric_feats = all_data.dtypes[all_data.dtypes != "object"].index
skewed_feats = train[numeric_feats].apply(lambda x: skew(x.dropna()))
skewed_feats = skewed_feats[skewed_feats > 0.75]
skewed_feats = skewed_feats.index
all_data[skewed_feats] = np.log1p(all_data[skewed_feats] )
oof_blend = np.zeros(len(train))
preds_knn = np.zeros(len(test))<define_variables>
f_drop_column = [
'BsmtFinSF1',
'LotFrontage',
'WoodDeckSF',
'2ndFlrSF',
'OpenPorchSF',
'HalfBath',
'LotArea',
'BsmtFullBath',
'BsmtUnfSF',
'BedroomAbvGr',
'ScreenPorch',
'PoolArea',
'MoSold',
'3SsnPorch',
'BsmtFinSF2',
'BsmtHalfBath',
'MiscVal',
'Id',
'LowQualFinSF',
'YrSold',
'OverallCond',
'MSSubClass',
'EnclosedPorch',
'KitchenAbvGr'
]
oof_blend = 0.5*(oof_qda3 + oof_qda4)
 preds_blend = 0.5*(preds_qda3 + preds_qda4 )<compute_test_metric>
MSZoning_map = {'RL':1, 'RM':0, 'C(all)':0, 'FV':1, 'RH':0}
Street_map = {"Pave":1, "Grvl":1}
Alley_map = {"Grvl":1, "Pave":1}
LotShape_map = {"Reg":0, "IR1":1, "IR2":1, "IR3":1}
LandContour_map = {"Lv1":0, "Bnk":0, "Low":1, "HLS":1}
Utilities_map = {"AllPub":1, "NoSeWa":1}
LotConfig_map = {"Inside":0, "FR2":0, "Corner":0, "CulDSac":1, "FR3":1}
LandSlope_map = {"Gtl":1, "Mod":1, "Sev":1}
Neighborhood_map = {
'CollgCr':1, 'Veenker':1, 'Crawfor':1, 'NoRidge':2, 'Mitchel':0, 'Somerst':1, 'NAmes':1, 'OldTown':0, 'BrkSide':0,
'Sawyer':0, 'NridgHt':2, 'NWAmes':0, 'SawyerW':1, 'IDOTRR':0, 'MeadowV':0, 'Edwards':0, 'Timber':1, 'Gilbert':1,
'StoneBr':2, 'ClearCr':1, 'NPkVill':0, 'Blmngtn':1, 'BrDale':0, 'SWISU':0, 'Blueste':0,
}
Condition1_map = {
"Norm":0, "Feedr":0, "PosN":1, "Artery":0, "RRAe":0, "RRNn":1, "RRAn":0, "PosA":1, "RRNe":0
}
Condition2_map = {
"Norm":0, "Artery":0, "RRNn":0, "Feedr":0, "PosN":1, "PosA":1, "RRAn":0, "RRAe":0,
}
BldgType_map = {
"1Fam":1, "2fmCon":1, "Duplex":1, "TwnhsE":1, "Twnhs":1
}
HouseStyle_map = {
'2Story':0, '1Story':0, '1.5Fin':0, '1.5Unf':0, 'SFoyer':0, 'SLvl':0, '2.5Unf':0, '2.5Fin':1
}
RoofStyle_map = {
'Gable':1, 'Hip':1, 'Gambrel':1, 'Mansard':1, 'Flat':1, 'Shed':1
}
RoofMatl_map = {
'CompShg':0, 'WdShngl':1, 'Metal':0, 'WdShake':0, 'Membran':0, 'Tar&Grv':0, 'Roll':0, 'ClyTile':0
}
Exterior1st_map = {
'VinylSd':1, 'MetalSd':0, 'Wd Sdng':0, 'HdBoard':0, 'BrkFace':1, 'WdShing':0,
'CemntBd':1, 'Plywood':0, 'AsbShng':0, 'Stucco':0, 'BrkComm':0, 'AsphShn':0,
'Stone':2, 'ImStucc':2, 'CBlock':0
}
Exterior2nd_map = {
'VinylSd':1, 'MetalSd':0, 'Wd Shng':0, 'HdBoard':0, 'Plywood':0, 'Wd Sdng':0,
'CmentBd':1, 'BrkFace':0, 'Stucco':0, 'AsbShng':0, 'Brk Cmn':0, 'ImStucc':1,
'AsphShn':0, 'Stone':0, 'Other':0, 'CBlock':0
}
MasVnrType_map = {
'BrkFace':0, 'None':0, 'Stone':1, 'BrkCmn':0
}
ExterQual_map = {
'Gd':0, 'TA':0, 'Ex':1, 'Fa':0
}
ExterCond_map = {
'TA':0, 'Gd':0, 'Fa':0, 'Po':0, 'Ex':1
}
Foundation_map = {
'PConc':1, 'CBlock':0, 'BrkTil':0, 'Wood':1, 'Slab':0, 'Stone':1
}
BsmtQual_map = {
'Gd':0, 'TA':0, 'Ex':1, 'Fa':0
}
BsmtCond_map = {
'TA':1, 'Gd':1, 'Fa':0, 'Po':0
}
BsmtExposure_map = {
'No':0, 'Gd':1, 'Mn':0, 'Av':0
}
BsmtFinType1_map = {
'GLQ':1, 'ALQ':0, 'Unf':0, 'Rec':0, 'BLQ':0, 'LwQ':0
}
BsmtFinType2_map = {
'Unf':0, 'BLQ':0, 'ALQ':1, 'Rec':0, 'LwQ':0, 'GLQ':1
}
Heating_map = {
'GasA':1, 'GasW':1, 'Grav':0, 'Wall':0, 'OthW':0, 'Floor':0
}
HeatingQC_map = {
'Ex':1, 'Gd':0, 'TA':0, 'Fa':0, 'Po':0
}
CentralAir_map = {
'Y':1, 'N':0
}
Electrical_map = {
'SBrkr':1, 'FuseF':0, 'FuseA':0, 'FuseP':0, 'Mix':0
}
KitchenQual_map = {
'Gd':0, 'TA':0, 'Ex':1, 'Fa':0
}
Functional_map = {
'Typ':1, 'Min1':1, 'Maj1':1, 'Min2':1, 'Mod':1, 'Maj2':0, 'Sev':1
}
FireplaceQu_map = {
'Gd':0, 'TA':0, 'Fa':0, 'Ex':1, 'Po':0
}
GarageType_map = {
'Attchd':1, 'Detchd':0, 'BuiltIn':1, 'CarPort':0, 'Basment':0, '2Types':0
}
GarageFinish_map = {
'RFn':0, 'Unf':0, 'Fin':1
}
GarageQual_map = {
'TA':0, 'Fa':0, 'Gd':1, 'Ex':1, 'Po':0
}
GarageCond_map = {
'TA':1, 'Fa':0, 'Gd':0, 'Po':0, 'Ex':0
}
PavedDrive_map = {
'Y':1, 'N':1, 'P':1
}
PoolQC_map = {
'Ex':1, 'Fa':0, 'Gd':0
}
Fence_map = {
'MnPrv':1, 'GdWo':1, 'GdPrv':1, 'MnWw':1
}
MiscFeature_map = {
'Shed':1, 'Gar2':1, 'Othr':0, 'TenC':1
}
SaleType_map = {
'WD':0, 'New':1, 'COD':0, 'ConLD':0, 'ConLI':0, 'CWD':0, 'ConLw':0, 'Con':1, 'Oth':0
}
SaleCondition_map = {
'Normal':0, 'Abnorml':0, 'Partial':1, 'AdjLand':0, 'Alloca':0, 'Family':0
}
print('Stacking model blend CV score =',round(roc_auc_score(train['target'], oof_blend),6))<save_to_csv>
all_data["MSZoning"] = all_data.MSZoning.replace(MSZoning_map)
 all_data["Street"] = all_data.Street.replace(Street_map)
 all_data["Alley"] = all_data.Alley.replace(Alley_map)
 all_data["LotShape"] = all_data.LotShape.replace(LotShape_map)
 all_data["LandContour"] = all_data.LandContour.replace(LandContour_map)
 all_data["Utilities"] = all_data.Utilities.replace(Utilities_map)
 all_data["LotConfig"] = all_data.LotConfig.replace(LotConfig_map)
 all_data["LandSlope"] = all_data.LandSlope.replace(LandSlope_map)
 all_data["Neighborhood"] = all_data.Neighborhood.replace(Neighborhood_map)
 all_data["Condition1"] = all_data.Condition1.replace(Condition1_map)
 all_data["Condition2"] = all_data.Condition2.replace(Condition2_map)
 all_data["BldgType"] = all_data.BldgType.replace(BldgType_map)
 all_data["HouseStyle"] = all_data.HouseStyle.replace(HouseStyle_map)
 all_data["RoofStyle"] = all_data.RoofStyle.replace(RoofStyle_map)
 all_data["RoofMatl"] = all_data.RoofMatl.replace(RoofMatl_map)
 all_data["Exterior1st"] = all_data.Exterior1st.replace(Exterior1st_map)
 all_data["Exterior2nd"] = all_data.Exterior2nd.replace(Exterior2nd_map)
 all_data["MasVnrType"] = all_data.MasVnrType.replace(MasVnrType_map)
 all_data["ExterQual"] = all_data.ExterQual.replace(ExterQual_map)
 all_data["ExterCond"] = all_data.ExterCond.replace(ExterCond_map)
 all_data["Foundation"] = all_data.Foundation.replace(Foundation_map)
 all_data["BsmtQual"] = all_data.BsmtQual.replace(BsmtQual_map)
 all_data["BsmtCond"] = all_data.BsmtCond.replace(BsmtCond_map)
 all_data["BsmtExposure"] = all_data.BsmtExposure.replace(BsmtExposure_map)
 all_data["BsmtFinType1"] = all_data.BsmtFinType1.replace(BsmtFinType1_map)
 all_data["BsmtFinType2"] = all_data.BsmtFinType2.replace(BsmtFinType2_map)
 all_data["Heating"] = all_data.Heating.replace(Heating_map)
 all_data["HeatingQC"] = all_data.HeatingQC.replace(HeatingQC_map)
 all_data["CentralAir"] = all_data.CentralAir.replace(CentralAir_map)
 all_data["Electrical"] = all_data.Electrical.replace(Electrical_map)
 all_data["KitchenQual"] = all_data.KitchenQual.replace(KitchenQual_map)
 all_data["Functional"] = all_data.Functional.replace(Functional_map)
 all_data["FireplaceQu"] = all_data.FireplaceQu.replace(FireplaceQu_map)
 all_data["GarageType"] = all_data.GarageType.replace(GarageType_map)
 all_data["GarageFinish"] = all_data.GarageFinish.replace(GarageFinish_map)
 all_data["GarageQual"] = all_data.GarageQual.replace(GarageQual_map)
 all_data["GarageCond"] = all_data.GarageCond.replace(GarageCond_map)
 all_data["PavedDrive"] = all_data.PavedDrive.replace(PavedDrive_map)
 all_data["PoolQC"] = all_data.PoolQC.replace(PoolQC_map)
 all_data["Fence"] = all_data.Fence.replace(Fence_map)
 all_data["MiscFeature"] = all_data.MiscFeature.replace(MiscFeature_map)
 all_data["SaleType"] = all_data.SaleType.replace(SaleType_map)
 all_data["SaleCondition"] = all_data.SaleCondition.replace(SaleCondition_map )
sub['target'] = preds_blend
sub.to_csv('submission_blend.csv', index=False )<prepare_output>
train['SalePrice'] = np.log1p(train['SalePrice'])
 y = train['SalePrice']
sub['target'] = preds_blend
sub['target'].hist(bins=100, alpha=0.6 )<define_variables>
all_data = pd.get_dummies(all_data)
 all_data.shape
oof_qda1 = oof_qda1.reshape(-1, 1)
 oof_qda2 = oof_qda2.reshape(-1, 1)
 oof_gmm = oof_gmm.reshape(-1, 1)
 oof_lr = oof_lr.reshape(-1, 1)
 oof_ls = oof_ls.reshape(-1, 1)
 oof_knn = oof_knn.reshape(-1, 1)
 oof_nn = oof_nn.reshape(-1, 1)
 oof_qda3 = oof_qda3.reshape(-1, 1 )<categorify>
drop_columns = [
'LandSlope',
'Street',
'FireplaceQu',
'Alley',
'LandContour_0',
'Utilities',
'MiscFeature',
'PavedDrive',
'SaleCondition',
'Fence',
'HouseStyle',
'BldgType',
'Condition2',
'RoofStyle',
'LandContour_Lvl',
'Foundation',
'MasVnrArea',
'GarageYrBlt',
'TotRmsAbvGrd',
'Electrical',
'GarageArea',
'YearBuilt',
'GarageType',
'YearRemodAdd',
'LandContour_1',
]
preds_qda1 = preds_qda1.reshape(-1, 1)
 preds_qda2 = preds_qda2.reshape(-1, 1)
 preds_gmm = preds_gmm.reshape(-1, 1)
 preds_lr = preds_lr.reshape(-1, 1)
 preds_ls = preds_ls.reshape(-1, 1)
 preds_knn = preds_knn.reshape(-1, 1)
 preds_nn = preds_nn.reshape(-1, 1)
 preds_qda3 = preds_qda3.reshape(-1, 1 )<concatenate>
all_data.drop('SalePrice', axis=1, inplace=True )
train_stck = np.concatenate([oof_qda1, oof_qda2, oof_gmm, oof_lr, oof_ls, oof_knn, oof_nn, oof_qda3], axis=1)
 test_stack = np.concatenate([preds_qda1, preds_qda2, preds_gmm, preds_lr, preds_ls, preds_knn, preds_nn, preds_qda3], axis=1 )<define_variables>
train = all_data[:len(train)]
test = all_data[len(train):]
y = y
oof_stack = np.zeros(len(train))
pred_stack = np.zeros(len(test))<predict_on_test>
model_xgb = xgb.XGBRegressor(max_depth=2,learning_rate=0.05,n_estimators=1000, min_child_weight=3, reg_lambda=2 )
for train_index, test_index in skf.split(train_stck, train['target']):
clf = LogisticRegression(solver='saga', penalty='l2', C=0.01, tol=0.001)
 clf.fit(train_stck[train_index], train['target'][train_index])
 oof_stack[test_index] = clf.predict_proba(train_stck[test_index,:])[:,1]
pred_stack += clf.predict_proba(test_stack)[:,1] / skf.n_splits<compute_test_metric>
model_xgb.fit(train, y )
print('{} model stack CV score ='.format(len(preds_list)) ,round(roc_auc_score(train['target'], oof_stack),6))<save_to_csv>
def predict_cv(model, train_x, train_y, test_x):
preds = []
preds_test = []
va_indexes = []
kf = KFold(n_splits=4, shuffle=True, random_state=1234)
 for i,(tr_idx, va_idx)in enumerate(kf.split(train_x)) :
tr_x, va_x = train_x.iloc[tr_idx], train_x.iloc[va_idx]
tr_y, va_y = train_y.iloc[tr_idx], train_y.iloc[va_idx]
model.fit(tr_x, tr_y)
 tr_pred = model.predict(tr_x)
 pred = model.predict(va_x)
 preds.append(pred)
 pred_test = model.predict(test_x)
 preds_test.append(pred_test)
 va_indexes.append(va_idx)
 print(' score Train : {:.6f}'.format(np.sqrt(mean_squared_error(tr_y, tr_pred))),
' score Valid : {:.6f}'.format(np.sqrt(mean_squared_error(va_y, pred))))
va_indexes = np.concatenate(va_indexes)
 preds = np.concatenate(preds, axis=0)
 order = np.argsort(va_indexes)
 pred_train = pd.DataFrame(preds[order])
 preds_test = pd.DataFrame(np.mean(preds_test, axis=0))
print('Score : {:.6f}'.format(np.sqrt(mean_squared_error(train_y, pred_train))))
return pred_train, preds_test, model
sub['target'] = pred_stack
sub.to_csv('submission_stack.csv', index=False )<set_options>
pred_train, preds_test, model = predict_cv(model_xgb, train, y, test)
warnings.filterwarnings('ignore' )<load_from_csv>
y_test = np.expm1(preds_test[0] )
<define_search_space><EOS> 
sub = pd.DataFrame()
sub['Id'] = TEST_ID
sub['SalePrice'] = y_test
sub.to_csv('submission.csv',index=False )