Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<save_to_csv>
warnings.filterwarnings('ignore' )
d_test = xgb.DMatrix(x_test)
 p_test = bst.predict(d_test)
 sub = pd.DataFrame()
sub['test_id'] = df_test['test_id']
sub['is_duplicate'] = p_test
sub.to_csv('simple_xgb.csv', index=False )<import_modules>
train = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/train.csv")
 test = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/test.csv")
 print(train.shape)
 print(test.shape )
from sklearn.feature_selection import VarianceThreshold<load_from_csv>
train = train[train.GrLivArea < 4500]
train.reset_index(drop = True, inplace = True)
 train['SalePrice'] = np.log1p(train['SalePrice'] )
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )<load_from_csv>
train.drop('Id', axis=1, inplace=True)
 test.drop('Id', axis=1, inplace=True)
 y = train['SalePrice'].reset_index(drop=True )
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )<define_variables>
all_data = pd.concat(( train, test)).reset_index(drop=True)
 all_data.drop('SalePrice', axis=1, inplace=True )
cols = [c for c in train.columns if c not in ['id', 'target', 'wheezy-copper-turtle-magic']]<create_dataframe>
missing_val_col = ["Alley",
"PoolQC",
"MiscFeature",
"Fence",
"FireplaceQu",
"GarageType",
"GarageFinish",
"GarageQual",
"GarageCond",
'BsmtQual',
'BsmtCond',
'BsmtExposure',
'BsmtFinType1',
'BsmtFinType2',
'MasVnrType']
for c in missing_val_col:
all_data[c] = all_data[c].fillna('None' )
def instant_model(train, test, cols = cols, clf = KNeighborsClassifier(leaf_size=2000, n_neighbors = 11, p = 2, weights = 'distance'), selection = "PCA"):
oof = np.zeros(len(train))
preds = np.zeros(len(test))
for i in tqdm(range(512)) :
train2 = train[train['wheezy-copper-turtle-magic'] == i]
test2 = test[test['wheezy-copper-turtle-magic'] == i]
idx1 = train2.index
idx2 = test2.index
data = pd.concat([pd.DataFrame(train2[cols]), pd.DataFrame(test2[cols])])
 if selection == "variance":
data2 = StandardScaler().fit_transform(VarianceThreshold(threshold=2 ).fit_transform(data[cols]))
train3 = pd.DataFrame(data2[:train2.shape[0]], index = idx1)
 test3 = pd.DataFrame(data2[train2.shape[0]:], index = idx2)
 elif selection == "PCA":
pca = PCA(n_components = 40, random_state= 1234)
 pca.fit(data[:train2.shape[0]])
 train3 = pd.DataFrame(pca.transform(data[:train2.shape[0]]), index = idx1)
 test3 = pd.DataFrame(pca.transform(data[train2.shape[0]:]), index = idx2)
 train3['target'] = train2['target']
skf = StratifiedKFold(n_splits=15, random_state=42)
 for train_index, test_index in skf.split(train3, train3['target']):
clf = clf
X_train = train3.iloc[train_index, :].drop(["target"], axis = 1)
 X_test = train3.iloc[test_index, :].drop(["target"], axis = 1)
 y_train = train3.iloc[train_index, :]['target']
y_test = train3.iloc[test_index, :]['target']
clf.fit(X_train, y_train)
 train_prob = clf.predict_proba(X_train)[:,1]
test_prob = clf.predict_proba(X_test)[:,1]
oof[idx1[test_index]] = test_prob
preds[idx2] += clf.predict_proba(test3)[:,1] / skf.n_splits
auc = roc_auc_score(train['target'], oof)
 print(f'AUC: {auc:.5}')
 return oof, preds, auc<feature_engineering>
missing_val_col2 = ['BsmtFinSF1',
'BsmtFinSF2',
'BsmtUnfSF',
'TotalBsmtSF',
'BsmtFullBath',
'BsmtHalfBath',
'GarageYrBlt',
'GarageArea',
'GarageCars',
'MasVnrArea']
for i in missing_val_col2:
all_data[i] = all_data[i].fillna(0)
 all_data['LotFrontage'] = all_data.groupby('Neighborhood')['LotFrontage'].transform(lambda x: x.fillna(x.mean()))
def get_newtrain(train, test, preds, oof):
test['target'] = preds
test.loc[test['target'] > 0.985, 'target'] = 1
test.loc[test['target'] < 0.015, 'target'] = 0
usefull_test = test[(test['target'] == 1)|(test['target'] == 0)]
print("No.of test records added: {}".format(usefull_test.shape[0]))
new_train = pd.concat([train, usefull_test] ).reset_index(drop=True)
 new_train.loc[oof > 0.985, 'target'] = 1
new_train.loc[oof < 0.015, 'target'] = 0
return new_train<train_model>
all_data['MSSubClass'] = all_data['MSSubClass'].astype(str)
 all_data['MSZoning'] = all_data.groupby('MSSubClass')['MSZoning'].transform(lambda x: x.fillna(x.mode() [0]))
all_data['Functional'] = all_data['Functional'].fillna('Typ')
 all_data['Utilities'] = all_data['Utilities'].fillna('AllPub')
 all_data['KitchenQual'] = all_data['KitchenQual'].fillna("TA")
 all_data['Electrical'] = all_data['Electrical'].fillna("SBrkr")
 all_data['Exterior1st'] = all_data['Exterior1st'].fillna(all_data['Exterior1st'].mode() [0])
 all_data['Exterior2nd'] = all_data['Exterior2nd'].fillna(all_data['Exterior2nd'].mode() [0])
 all_data['SaleType'] = all_data['SaleType'].fillna(all_data['SaleType'].mode() [0] )
newtrain = get_newtrain(train, test, preds, oof)
 oof_qd, preds_qd, auc = instant_model(newtrain, test, clf = QuadraticDiscriminantAnalysis(0.5),selection="variance" )<train_model>
all_data = fixing_skewness(all_data)
 all_data = all_data.drop(['Utilities', 'Street', 'PoolQC',], axis=1)
 all_data['TotalSF'] = all_data['TotalBsmtSF'] + all_data['1stFlrSF'] + all_data['2ndFlrSF']
all_data['YrBltAndRemod']=all_data['YearBuilt']+all_data['YearRemodAdd']
all_data['Total_sqr_footage'] =(all_data['BsmtFinSF1'] + all_data['BsmtFinSF2'] +
all_data['1stFlrSF'] + all_data['2ndFlrSF'])
 all_data['Total_Bathrooms'] =(all_data['FullBath'] +(0.5 * all_data['HalfBath'])+
all_data['BsmtFullBath'] +(0.5 * all_data['BsmtHalfBath']))
all_data['Total_porch_sf'] =(all_data['OpenPorchSF'] + all_data['3SsnPorch'] +
all_data['EnclosedPorch'] + all_data['ScreenPorch'] +
all_data['WoodDeckSF'])
 all_data['haspool'] = all_data['PoolArea'].apply(lambda x: 1 if x > 0 else 0)
 all_data['has2ndfloor'] = all_data['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)
 all_data['hasgarage'] = all_data['GarageArea'].apply(lambda x: 1 if x > 0 else 0)
 all_data['hasbsmt'] = all_data['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 all_data['hasfireplace'] = all_data['Fireplaces'].apply(lambda x: 1 if x > 0 else 0 )
for i in range(1):
newtrain = get_newtrain(newtrain, test, preds_qd, oof_qd)
 oof_qd, preds_qd, auc = instant_model(newtrain, test, clf = QuadraticDiscriminantAnalysis(0.5),selection="variance" )<save_to_csv>
all_data = pd.get_dummies(all_data ).reset_index(drop = True)
 all_data.shape
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub['target'] = preds_qd
sub.to_csv('submission.csv',index=False)
<load_from_csv>
X = all_data.iloc[:len(y), :]
X_sub = all_data.iloc[len(y):, :]
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 train.head()<data_type_conversions>
outliers = [30, 88, 462, 631, 1322]
X = X.drop(X.index[outliers])
 y = y.drop(y.index[outliers] )
def get_mean_cov(x,y):
model = GraphicalLasso()
ones =(y==1 ).astype(bool)
 x2 = x[ones]
model.fit(x2)
 p1 = model.precision_
m1 = model.location_
onesb =(y==0 ).astype(bool)
 x2b = x[onesb]
model.fit(x2b)
 p2 = model.precision_
m2 = model.location_
ms = np.stack([m1,m2])
 ps = np.stack([p1,p2])
 return ms,ps 
<sort_values>
overfitted_features = overfit_reducer(X)
 X = X.drop(overfitted_features, axis=1)
 X_sub = X_sub.drop(overfitted_features, axis=1 )
cols = [c for c in train.columns if c not in ['id', 'target']]
cols.remove('wheezy-copper-turtle-magic')
 oof = np.zeros(len(train))
preds = np.zeros(len(test))
for i in tqdm(range(512)) :
train2 = train[train['wheezy-copper-turtle-magic']==i]
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train2.index; idx2 = test2.index
train2.reset_index(drop=True,inplace=True)
 sel = VarianceThreshold(threshold=1.5 ).fit(train2[cols])
 train3 = sel.transform(train2[cols])
 test3 = sel.transform(test2[cols])
 skf = StratifiedKFold(n_splits=11, random_state=42, shuffle=True)
 for train_index, test_index in skf.split(train3, train2['target']):
ms, ps = get_mean_cov(train3[train_index,:],train2.loc[train_index]['target'].values)
 gm = GaussianMixture(n_components=2, init_params='random', covariance_type='full', tol=0.001,reg_covar=0.001, max_iter=100, n_init=1,means_init=ms, precisions_init=ps)
 gm.fit(np.concatenate([train3[train_index,:],test3],axis = 0))
oof[idx1[test_index]] = gm.predict_proba(train3[test_index,:])[:,0]
preds[idx2] += gm.predict_proba(test3)[:,0] / skf.n_splits
auc = roc_auc_score(train['target'],oof)
 print('QDA scores CV =',round(auc,5))<save_to_csv>
kfolds = KFold(n_splits=10, shuffle=True, random_state=42)
 def rmse(y, y_hat):
return np.sqrt(mean_squared_error(y, y_hat))
def rmse_cv(model, X = X):
return np.sqrt(-1 * cross_val_score(model, X, y, scoring="neg_mean_squared_error", cv=kfolds))
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub['target'] = preds
sub.to_csv('submission.csv',index=False)
 plt.hist(preds,bins=100)
 plt.title('Final Test.csv predictions')
 plt.show()<load_from_csv>
alphas_rid = [14.5, 14.6, 14.7, 14.8, 14.9, 15, 15.1, 15.2, 15.3, 15.4, 15.5]
alphas_las = [5e-05, 0.0001, 0.0002, 0.0003, 0.0004, 0.0005, 0.0006, 0.0007, 0.0008]
e_alphas = [0.0001, 0.0002, 0.0003, 0.0004, 0.0005, 0.0006, 0.0007]
e_l1ratio = [0.8, 0.85, 0.9, 0.95, 0.99, 1]
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 train.head()<train_on_grid>
ridge = make_pipeline(RobustScaler() , RidgeCV(alphas=alphas_rid, cv=kfolds))
lasso = make_pipeline(RobustScaler() , LassoCV(max_iter=1e7, alphas=alphas_las, random_state=42, cv=kfolds))
elasticnet = make_pipeline(RobustScaler() , ElasticNetCV(max_iter=1e7, alphas=e_alphas, cv=kfolds, l1_ratio=e_l1ratio))
svr = make_pipeline(RobustScaler() , SVR(C= 20, epsilon= 0.008, gamma=0.0003,))
lightgbm = LGBMRegressor(objective='regression',
num_leaves=4,
learning_rate=0.01,
n_estimators=5000,
max_bin=200,
bagging_fraction=0.75,
bagging_freq=5,
bagging_seed=7,
feature_fraction=0.2,
feature_fraction_seed=7,
verbose=-1, 
)
 xgboost = XGBRegressor(learning_rate=0.01,n_estimators=3460,
max_depth=3, min_child_weight=0,
gamma=0, subsample=0.7,
colsample_bytree=0.7,
objective='reg:squarederror', nthread=-1,
scale_pos_weight=1, seed=27,
reg_alpha=0.00006)
 stack_gen = StackingCVRegressor(regressors=(ridge, lasso, elasticnet, xgboost, lightgbm),
meta_regressor=xgboost,
use_features_in_secondary=True )
cols = [c for c in train.columns if c not in ['id', 'target']]
cols.remove('wheezy-copper-turtle-magic')
 oof = np.zeros(len(train))
preds = np.zeros(len(test))
for i in range(512):
train2 = train[train['wheezy-copper-turtle-magic']==i]
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train2.index; idx2 = test2.index
train2.reset_index(drop=True,inplace=True)
 sel = VarianceThreshold(threshold=1.5 ).fit(train2[cols])
 train3 = sel.transform(train2[cols])
 test3 = sel.transform(test2[cols])
 skf = StratifiedKFold(n_splits=11, random_state=42, shuffle=True)
 for train_index, test_index in skf.split(train3, train2['target']):
clf = QuadraticDiscriminantAnalysis(reg_param=0.5)
 clf.fit(train3[train_index,:],train2.loc[train_index]['target'])
 oof[idx1[test_index]] = clf.predict_proba(train3[test_index,:])[:,1]
preds[idx2] += clf.predict_proba(test3)[:,1] / skf.n_splits
auc = roc_auc_score(train['target'],oof)
 print('QDA scores CV =',round(auc,5))<feature_engineering>
score = rmse_cv(ridge)
 print("Ridge: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() ,)
 score = rmse_cv(lasso)
 print("Lasso: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() ,
score = rmse_cv(elasticnet)
 print("Elastic Net: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() ,)
 score = rmse_cv(svr)
 print("SVR: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() ,)
 score = rmse_cv(lightgbm)
 print("lightgmb: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() ,)
 score = rmse_cv(xgboost)
 print("XGBoost: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() , )
test['target'] = preds
oof = np.zeros(len(train))
preds = np.zeros(len(test))
for k in range(512):
train2 = train[train['wheezy-copper-turtle-magic']==k]
train2p = train2.copy() ; idx1 = train2.index
test2 = test[test['wheezy-copper-turtle-magic']==k]
test2p = test2[(test2['target']<=0.01)|(test2['target']>=0.99)].copy()
test2p.loc[ test2p['target']>=0.5, 'target' ] = 1
test2p.loc[ test2p['target']<0.5, 'target' ] = 0
train2p = pd.concat([train2p,test2p],axis=0)
 train2p.reset_index(drop=True,inplace=True)
 sel = VarianceThreshold(threshold=1.5 ).fit(train2p[cols])
 train3p = sel.transform(train2p[cols])
 train3 = sel.transform(train2[cols])
 test3 = sel.transform(test2[cols])
 skf = StratifiedKFold(n_splits=11, random_state=42, shuffle=True)
 for train_index, test_index in skf.split(train3p, train2p['target']):
test_index3 = test_index[ test_index<len(train3)]
clf = QuadraticDiscriminantAnalysis(reg_param=0.5)
 clf.fit(train3p[train_index,:],train2p.loc[train_index]['target'])
 oof[idx1[test_index3]] = clf.predict_proba(train3[test_index3,:])[:,1]
preds[test2.index] += clf.predict_proba(test3)[:,1] / skf.n_splits
auc = roc_auc_score(train['target'],oof)
 print('Pseudo Labeled QDA scores CV =',round(auc,5))<save_to_csv>
stack_gen_model = stack_gen.fit(np.array(X), np.array(y))
elastic_model_full_data = elasticnet.fit(X, y)
 lasso_model_full_data = lasso.fit(X, y)
 ridge_model_full_data = ridge.fit(X, y)
 svr_model_full_data = svr.fit(X, y)
 xgb_model_full_data = xgboost.fit(X, y)
 lgb_model_full_data = lightgbm.fit(X, y )
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub['target'] = preds
sub.to_csv('submission.csv',index=False)
 plt.hist(preds,bins=100)
 plt.title('Final Test.csv predictions')
 plt.show()<set_options>
def blend_models_predict(X):
return(( 0.1 * elastic_model_full_data.predict(X)) + \ 
(0.05 * lasso_model_full_data.predict(X)) + \ 
(0.2 * ridge_model_full_data.predict(X)) + \ 
(0.1 * svr_model_full_data.predict(X)) + \ 
(0.15 * xgb_model_full_data.predict(X)) + \ 
(0.1 * lgb_model_full_data.predict(X)) + \ 
(0.3 * stack_gen_model.predict(np.array(X))))
warnings.filterwarnings('ignore' )<load_from_csv>
print('RMSLE score on train data:')
 print(rmse(y, blend_models_predict(X)) )
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 cols = [c for c in train.columns if c not in ['id', 'target', 'wheezy-copper-turtle-magic']]<create_dataframe>
print('Predict submission')
 submission = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/sample_submission.csv")
 submission.iloc[:,1] = np.floor(np.expm1(blend_models_predict(X_sub)) )
oof = np.zeros(len(train))
preds = np.zeros(len(test))
for i in tqdm_notebook(range(512)) :
train2 = train[train['wheezy-copper-turtle-magic']==i]
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train2.index; idx2 = test2.index
train2.reset_index(drop=True,inplace=True)
 data = pd.concat([pd.DataFrame(train2[cols]), pd.DataFrame(test2[cols])])
 data2 = VarianceThreshold(threshold=2 ).fit_transform(data[cols])
 train3 = data2[:train2.shape[0]]; test3 = data2[train2.shape[0]:]
skf = StratifiedKFold(n_splits=11, random_state=42)
 for train_index, test_index in skf.split(train2, train2['target']):
clf = QuadraticDiscriminantAnalysis(0.5)
 clf.fit(train3[train_index,:],train2.loc[train_index]['target'])
 oof[idx1[test_index]] = clf.predict_proba(train3[test_index,:])[:,1]
preds[idx2] += clf.predict_proba(test3)[:,1] / skf.n_splits
auc = roc_auc_score(train['target'], oof)
 print(f'AUC: {auc:.5}' )<feature_engineering>
print('Blend with Top Kernels submissions
')
 sub_1 = pd.read_csv(".. /input/top-sub/S1.csv")
 sub_2 = pd.read_csv(".. /input/top-sub/S2.csv")
 sub_3 = pd.read_csv(".. /input/top-sub/S3.csv")
 submission.iloc[:,1] = np.floor(( 0.25 * np.floor(np.expm1(blend_models_predict(X_sub)))) + 
(0.25 * sub_1.iloc[:,1])+ 
(0.25 * sub_2.iloc[:,1])+ 
(0.25 * sub_3.iloc[:,1]))
test['target'] = preds
test.loc[test['target'] > 0.99, 'target'] = 1
test.loc[test['target'] < 0.01, 'target'] = 0<feature_engineering>
q1 = submission['SalePrice'].quantile(0.005)
 q2 = submission['SalePrice'].quantile(0.995)
 submission['SalePrice'] = submission['SalePrice'].apply(lambda x: x if x > q1 else x*0.77)
 submission['SalePrice'] = submission['SalePrice'].apply(lambda x: x if x < q2 else x*1.1)
 submission.to_csv("submission.csv", index=False )