Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<train_on_grid>
train = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/train.csv")
 test = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/test.csv")
 train["Test"]=0
test["Test"]=1
X_all = pd.concat(( train, test),sort=True ).reset_index(drop=True)
 X_all.loc[:,"SalePrice"] = np.log(X_all["SalePrice"] )
grid_hard = ensemble.VotingClassifier(estimators = vote_est , voting = 'hard')
 grid_hard_cv = model_selection.cross_validate(grid_hard, X_train[FEATURES], y_train, cv = cv_split, return_train_score=True)
 grid_hard.fit(X_train[FEATURES], y_train)
 print("Hard Voting w/Tuned Hyperparameters Training w/bin score mean: {:.2f}".format(grid_hard_cv['train_score'].mean() *100))
print("Hard Voting w/Tuned Hyperparameters Test w/bin score mean: {:.2f}".format(grid_hard_cv['test_score'].mean() *100))
print("Hard Voting w/Tuned Hyperparameters Test w/bin score 3*std: +/- {:.2f}".format(grid_hard_cv['test_score'].std() *100*3))
print('-'*10)
 grid_soft = ensemble.VotingClassifier(estimators = vote_est , voting = 'soft')
 grid_soft_cv = model_selection.cross_validate(grid_soft, X_train[FEATURES], y_train, cv = cv_split, return_train_score=True)
 grid_soft.fit(X_train[FEATURES], y_train)
 print("Soft Voting w/Tuned Hyperparameters Training w/bin score mean: {:.2f}".format(grid_soft_cv['train_score'].mean() *100))
print("Soft Voting w/Tuned Hyperparameters Test w/bin score mean: {:.2f}".format(grid_soft_cv['test_score'].mean() *100))
print("Soft Voting w/Tuned Hyperparameters Test w/bin score 3*std: +/- {:.2f}".format(grid_soft_cv['test_score'].std() *100*3))
print('-'*10)
<save_to_csv>
summary = pd.DataFrame({"Missing":len(X_all)- X_all.count() ,"Typical":np.nan})
 for col in summary[(summary.Missing > 0)].index:
if col == "SalePrice": continue
colmode=X_all[col].mode()
summary.loc[col,"Typical"]=colmode[0]
X_all[col].fillna(colmode[0],inplace=True)
 summary[summary.Missing > 0]
submission_df = pd.DataFrame(columns=['PassengerId', 'Survived'])
 submission_df['PassengerId'] = df_test['PassengerId']
submission_df['Survived'] = grid_hard.predict(X_test[FEATURES])
 submission_df["Survived"] = submission_df["Survived"].astype(int)
 submission_df.to_csv('submission.csv', header=True, index=False)
 submission_df.head(10 )<set_options>
functional_map = {"Sal":1,"Sev":0.3,"Maj1":0.1,"Maj2":0.15,"Mod":0.1,"Min1":0.05,"Min2":0.05,"Typ":0}
quality_map = {"Ex":5,"Gd":4,"TA":3,"Fa":2,"Po":1,"NA":0,0:0}
exposure_map = {"Gd":3,"Av":1,"Mn":0,"No":0,"NA":0}
lotqual_map = {"CulDSac":2,"Inside":0,"Corner":1,"FR2": -1,"FR3":-2}
def feature_eng(X,num_to_categ={},full_data=False):
new_columns= ["SF","BaseGood","BaseTA","Age","Pool_Ind","New","SaleCondition_Abn","Res","Pred"]
for el in new_columns: X.insert(1,el,0)
 X.loc[(X.SaleType=="New")&(X.YearBuilt >= X.YrSold-1),"New"]= 1
X.loc[X.SaleCondition == "Abnorml","SaleCondition_Abn"]=1
X.loc[(X.PoolArea > 0),"Pool_Ind"]= 1
qual_cols = ["ExterQual","ExterCond","BsmtQual","BsmtCond","HeatingQC","KitchenQual","FireplaceQu","GarageQual"]
for col in qual_cols: X.loc[:,col]= X[col].map(quality_map)
 X["LotConfig"]=X["LotConfig"].map(lotqual_map)
 X["Functional"]=X["Functional"].map(functional_map)
 X["Age"]= np.maximum(( X["YrSold"]-X["YearBuilt"])/10,0)
 X["QualityScore"]=(X.KitchenQual * 2.5 + X.FireplaceQu *1 + X.GarageQual* 1.5 + X.HeatingQC * 1.5 + 2* X.BsmtQual)/8.5-3
X["OverallCond"]=(X.OverallCond-5)
 X["OverallQual"]=(X.OverallQual-5)
 X.loc[X.BsmtFinType1.isin({"GLQ","ALQ"}),"BaseGood"]+= X.loc[X.BsmtFinType1.isin({"GLQ","ALQ"}),"BsmtFinSF1"]
X.loc[X.BsmtFinType1.isin({"BLQ","Rec","LwQ"}),"BaseTA"]+= X.loc[X.BsmtFinType1.isin({"BLQ","Rec","LwQ"}),"BsmtFinSF1"]
X.loc[X.BsmtFinType2.isin({"GLQ","ALQ"}),"BaseGood"]+= X.loc[X.BsmtFinType2.isin({"GLQ","ALQ"}),"BsmtFinSF2"]
X.loc[X.BsmtFinType2.isin({"BLQ","Rec","LwQ"}),"BaseTA"]+= X.loc[X.BsmtFinType2.isin({"BLQ","Rec","LwQ"}),"BsmtFinSF2"]
X["SF"]= X.GrLivArea - 0.1*X["2ndFlrSF"]+(X.BaseGood*.65+X.BaseTA*0.5)+0.25*X.BsmtUnfSF
X["BsmtExposure"]= X["BsmtExposure"].map(exposure_map)*(( X.BaseGood+X.BaseTA)/X.SF)
 X["SFLog"]=np.log(X.SF)
 X["LotArea"]=np.log(X.LotArea)- np.log(X.SF)
 if not full_data:
X=X[["Res","Pred","SalePrice","Test","Id"] + ["SFLog","Functional","OverallQual","OverallCond",
"LotArea","GarageCars","MSZoning","Neighborhood","New","Age","BldgType","SaleCondition_Abn",
"Pool_Ind","BsmtExposure","QualityScore","LotConfig","Condition1"]]
X = pd.get_dummies(X,columns=list(num_to_categ))
X = pd.get_dummies(X)
 for el in set(X.columns ).difference({"Test","Pred","Res"}):
col_mode=X.loc[X.Test==0,el].mode() [0]
if len(X.loc[(X.Test == 0)&(X[el] !=col_mode),"SFLog"])< 5:
X.drop(columns=el, inplace = True)
 print("Feature removed:",el)
 inter_features ={"SFLog","Age","OverallCond","OverallQual","QualityScore","Functional"}.difference(num_to_categ)
 poly = PolynomialFeatures(interaction_only=True,include_bias=False)
 inter_cols = poly.fit_transform(X[inter_features])
 X1= pd.DataFrame(inter_cols,columns= poly.get_feature_names(list(inter_features)) ,index=X.index)
 X = pd.concat([X1,X.drop(columns=inter_features)],axis=1)
 return X
%matplotlib inline
%config InlineBackend.figure_format = 'retina'
plt.style.use('ggplot')
 warnings.filterwarnings('ignore')
 plt.rc('font', size=18)
 plt.rc('axes', titlesize=22)
 plt.rc('axes', labelsize=18)
 plt.rc('xtick', labelsize=12)
 plt.rc('ytick', labelsize=12)
 plt.rc('legend', fontsize=12)
 plt.rcParams['font.sans-serif'] = ['Verdana']
pd.options.mode.chained_assignment = None
pd.options.display.max_seq_items = 500
pd.options.display.max_rows = 500
pd.set_option('display.float_format', lambda x: '%.5f' % x)
 BASE_PATH = "/kaggle/input/house-prices-advanced-regression-techniques/"<load_from_csv>
def model_fit(model,X,target_col,folds=5):
kf = KFold(folds, shuffle=True, random_state=4991)
 drop_cols = ["Test","Id","Pred","Res",target_col]
X_train = X[X.Test==0].drop(columns=drop_cols)
 y_train = X.loc[X.Test==0,target_col]
model.fit(X_train,y_train)
 X["Pred"] = model.predict(X.drop(columns=drop_cols))
X["Res"]= X.Pred-X[target_col]
score =(-cross_val_score(model, X_train, y_train, scoring="neg_mean_squared_error", cv = kf)) **0.5
df = pd.DataFrame({"Coeff":model.coef_,"Y mean":(X_train*model.coef_ ).mean() ,"Y std":(X_train*model.coef_ ).std() } ).sort_values("Y std",ascending=False)
 return X, df, score
X=X_all[~X_all.Id.isin(outliers)].copy()
model = Lasso(alpha=0.0003,random_state=3591,max_iter=100000,fit_intercept=True)
 X = feature_eng(X,full_data=False)
 X, df, score = model_fit(model,X,"SalePrice")
 print("
CV score: ",score,"
Mean: {:.4f} Std: {:.4f}
".format(score.mean() , score.std()))
df.head(10 )
df = pd.read_csv(f"{BASE_PATH}train.csv")
 df_test = pd.read_csv(f"{BASE_PATH}test.csv")
 feat = pd.concat([df, df_test] ).reset_index(drop=True ).copy()<data_type_conversions>
outliers+=[589,633,463,813,1433,411,804]
X=X_all[~X_all.Id.isin(outliers)].copy()
X = feature_eng(X)
 X, df, score = model_fit(model,X,"SalePrice")
 print("
Model RMSLE CV score: ",score,"
Mean: {:.4f} Std: {:.4f}
".format(score.mean() , score.std()))
df.head(10 )
feat.Alley = feat.Alley.fillna("None")
 bsmt_cols = ['BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2', 'BsmtQual']
for col in bsmt_cols:
feat[col] = feat[col].fillna("None")
 for col in ['BsmtFullBath', 'BsmtHalfBath', 'BsmtUnfSF', 'BsmtFinSF1', 'BsmtFinSF2', 'TotalBsmtSF']:
feat[col] = feat[col].fillna(0 ).astype(int)
 feat.Electrical = feat.Electrical.fillna("SBrkr")
 feat.Exterior1st = feat.Exterior1st.fillna("VinylSd")
 feat.Exterior2nd = feat.Exterior2nd.fillna("None")
 feat.Fence = feat.Fence.fillna("None")
 feat.FireplaceQu = feat.FireplaceQu.fillna("None")
 feat.Functional = feat.Functional.fillna("Typ")
 grg_cols = ['GarageCond', 'GarageFinish', 'GarageQual', 'GarageType']
for col in grg_cols:
feat[col] = feat[col].fillna("None")
 for col in ['GarageArea', 'GarageCars', 'GarageYrBlt']:
feat[col] = feat[col].fillna(0 ).astype(int)
 to_fix = feat[feat.GarageYrBlt == 2207].index
feat.loc[to_fix, "GarageYrBlt"] = int(feat.GarageYrBlt.mean())
 feat.KitchenQual = feat.KitchenQual.fillna("TA")
 feat.LotFrontage = feat.LotFrontage.fillna(0 ).astype(int)
 feat.MSZoning = feat.MSZoning.fillna("RL")
 feat.MSSubClass = feat.MSSubClass.astype(str)
 feat.MasVnrType = feat.MasVnrType.fillna("None")
 feat.MasVnrArea = feat.MasVnrArea.fillna(0 ).astype(int)
 feat.MiscFeature = feat.MiscFeature.fillna("None")
 feat.PoolQC = feat.PoolQC.fillna("None")
 feat.SaleType = feat.SaleType.fillna("WD")
 feat.Utilities = feat.Utilities.fillna("AllPub" )<categorify>
model = Lasso(alpha=0.00033,random_state=3591,max_iter=100000,fit_intercept=True)
 X=X_all[~X_all.Id.isin(outliers)].copy()
X = feature_eng(X,full_data=True)
 X, df, score = model_fit(model,X,"SalePrice")
 print("
Model RMSLE CV score: ",score,"
Mean: {:.4f} Std: {:.4f}
".format(score.mean() , score.std()))
df.head(10 )
feat = feat.replace({ "Alley": {"None" : 0, "Grvl" : 1, "Pave" : 2},
"BsmtCond": {"None" : 0, "Po" : 1, "Fa" : 2, "TA" : 3,
"Gd" : 4, "Ex" : 5},
"BsmtExposure": {"None" : 0, "No" : 2, "Mn" : 2, "Av": 3,
"Gd" : 4},
"BsmtFinType1": {"None" : 0, "Unf" : 1, "LwQ": 2, "Rec" : 3,
"BLQ" : 4, "ALQ" : 5, "GLQ" : 6},
"BsmtFinType2": {"None" : 0, "Unf" : 1, "LwQ": 2, "Rec" : 3,
"BLQ" : 4,
"ALQ" : 5, "GLQ" : 6},
"BsmtQual": {"None" : 0, "Po" : 1, "Fa" : 2, "TA": 3,
"Gd" : 4, "Ex" : 5},
"CentralAir": {"None" : 0, "N" : 1, "Y" : 2},
"ExterCond": {"None" : 0, "Po" : 1, "Fa" : 2, "TA": 3,
"Gd": 4, "Ex" : 5},
"ExterQual": {"None" : 0, "Po" : 1, "Fa" : 2, "TA": 3,
"Gd": 4, "Ex" : 5},
"Fence": {"None" : 0, "MnWw" : 1, "GdWo" : 2, "MnPrv": 3,
"GdPrv" : 4},
"FireplaceQu": {"None" : 0, "Po" : 1, "Fa" : 2, "TA" : 3,
"Gd" : 4, "Ex" : 5},
"Functional": {"None" : 0, "Sal" : 1, "Sev" : 2, "Maj2" : 3,
"Maj1" : 4, "Mod": 5, "Min2" : 6, "Min1" : 7,
"Typ" : 8},
"GarageCond": {"None" : 0, "Po" : 1, "Fa" : 2, "TA" : 3,
"Gd" : 4, "Ex" : 5},
"GarageQual": {"None" : 0, "Po" : 1, "Fa" : 2, "TA" : 3,
"Gd" : 4, "Ex" : 5},
"GarageFinish": {"None" : 0, "Unf" : 1, "RFn" : 2, "Fin" : 3},
"HeatingQC": {"None" : 0, "Po" : 1, "Fa" : 2, "TA" : 3,
"Gd" : 4, "Ex" : 5},
"KitchenQual": {"None" : 0, "Po" : 1, "Fa" : 2, "TA" : 3,
"Gd" : 4, "Ex" : 5},
"LandContour": {"None" : 0, "Low" : 1, "HLS" : 2, "Bnk" : 3,
"Lvl" : 4},
"LandSlope": {"None" : 0, "Sev" : 1, "Mod" : 2, "Gtl" : 3},
"LotShape": {"None" : 0, "IR3" : 1, "IR2" : 2, "IR1" : 3,
"Reg" : 4},
"PavedDrive": {"None" : 0, "N" : 0, "P" : 1, "Y" : 2},
"PoolQC": {"None" : 0, "Fa" : 1, "TA" : 2, "Gd" : 3,
"Ex" : 4},
"Street": {"None" : 0, "Grvl" : 1, "Pave" : 2},
"Utilities": {"None" : 0, "ELO" : 1, "NoSeWa" : 2, "NoSewr" : 3,
"AllPub" : 4}} 
)
 feat.BsmtCond = feat.BsmtCond.astype(int )<categorify>
sub = pd.DataFrame({"Id":X.loc[X.Test==1,"Id"],"SalePrice":np.exp(X.loc[X.Test==1,"Pred"])})
 sub.to_csv('submissionV1.1.csv',index=False )
def onehot_encode(data):
df_numeric = data.select_dtypes(exclude=['object'])
 df_obj = data.select_dtypes(include=['object'] ).copy()
cols = []
for c in df_obj:
dummies = pd.get_dummies(df_obj[c])
 dummies.columns = [c + "_" + str(x)for x in dummies.columns]
cols.append(dummies)
 df_obj = pd.concat(cols, axis=1)
 data = pd.concat([df_numeric, df_obj], axis=1)
 data.reset_index(inplace=True, drop=True)
 return data
feat = onehot_encode(feat )<categorify>
X=X_all[~X_all.Id.isin(outliers)].copy()
X = feature_eng(X,["Functional"])
 X, df, score = model_fit(model,X,"SalePrice")
 df.loc[["Functional_0.0","Functional_0.05","Functional_0.15","Functional_0.1"],"Coeff"]-df.loc["Functional_0.0","Coeff"]