Kaggle competition: House Prices - Advanced Regression Techniques
df = df.apply(lambda x:x.fillna(x.value_counts().index[0]))<create_dataframe>
pd.options.display.max_rows = 1000
pd.options.display.max_columns = 20
df = df.fillna(df.median())
 df_missing =(df.isnull().sum() / len(df)) * 100
df_missing = df_missing.drop(df_missing[df_missing == 0].index ).sort_values(ascending=False)[:30]
missing_data = pd.DataFrame({'Missing Ratio' :df_missing})
 missing_data.head(30 )<categorify>
import matplotlib
import matplotlib.pyplot as plt
import seaborn as sns
from scipy.stats import skew
from sklearn.model_selection import train_test_split, KFold
from sklearn.linear_model import ElasticNet
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import RobustScaler
from sklearn.metrics import mean_squared_error
df= pd.get_dummies(df, columns= ['MSZoning', 'Street', 'Alley', 'LotShape', 'LandContour', 'Utilities',
'LotConfig', 'LandSlope', 'Neighborhood', 'Condition1', 'Condition2',
'BldgType', 'HouseStyle', 'RoofStyle', 'RoofMatl', 'Exterior1st',
'Exterior2nd', 'MasVnrType', 'ExterQual', 'ExterCond', 'Foundation',
'BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2',
'Heating', 'HeatingQC', 'CentralAir', 'Electrical', 'KitchenQual',
'Functional', 'FireplaceQu', 'GarageType', 'GarageFinish', 'GarageQual',
'GarageCond', 'PavedDrive', 'PoolQC', 'Fence', 'MiscFeature',
'SaleType', 'SaleCondition'] )<drop_column>
train = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 test = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv' )
object_columns_df = object_columns_df.drop(['Heating','RoofMatl','Condition2','Street','Utilities'],axis=1)
<filter>
features = ['GrLivArea','OverallQual','2ndFlrSF','TotalBsmtSF','1stFlrSF','GarageCars','YearBuilt', 'GarageArea','ExterQual',
'YearRemodAdd','FireplaceQu','GarageYrBlt','FullBath','MSSubClass','LotArea','Fireplaces','KitchenQual','MSZoning',
'GarageType','BsmtFinSF1','Neighborhood','BsmtQual','TotRmsAbvGrd','HalfBath','BldgType','GarageFinish',
'Foundation','BedroomAbvGr','HouseStyle','CentralAir','OpenPorchSF','HeatingQC','BsmtFinType1','BsmtUnfSF',
'GarageCond','GarageQual','KitchenAbvGr','OverallCond','BsmtCond','MasVnrArea','BsmtFullBath','Exterior1st',
'Exterior2nd','PavedDrive','WoodDeckSF','LandContour','MasVnrType','BsmtFinType2','Functional','Fence']
Negatif = numerical_columns_df[numerical_columns_df['Age_House'] < 0]
Negatif 
<feature_engineering>
train = train[features+['SalePrice']]
test = test[['Id']+features]
all_data = pd.concat(( train.loc[:,'GrLivArea':'Fence'],
test.loc[:,'GrLivArea':'Fence']))
all_data['SalePrice'] = train['SalePrice']
numerical_columns_df['TotalBsmtBath'] = numerical_columns_df['BsmtFullBath'] + numerical_columns_df['BsmtFullBath']*0.5
numerical_columns_df['TotalBath'] = numerical_columns_df['FullBath'] + numerical_columns_df['HalfBath']*0.5
numerical_columns_df['TotalSA']=numerical_columns_df['TotalBsmtSF'] + numerical_columns_df['1stFlrSF'] + numerical_columns_df['2ndFlrSF'] 
<categorify>
obj_list = []
for i in all_data.columns :
if all_data[i].dtypes == 'object':
obj_list.append(i )
bin_map = {'TA':2,'Gd':3, 'Fa':1,'Ex':4,'Po':1,'None':0,'Y':1,'N':0,'Reg':3,'IR1':2,'IR2':1,'IR3':0,"None" : 0,
"No" : 2, "Mn" : 2, "Av": 3,"Gd" : 4,"Unf" : 1, "LwQ": 2, "Rec" : 3,"BLQ" : 4, "ALQ" : 5, "GLQ" : 6
}
object_columns_df['ExterQual'] = object_columns_df['ExterQual'].map(bin_map)
 object_columns_df['ExterCond'] = object_columns_df['ExterCond'].map(bin_map)
 object_columns_df['BsmtCond'] = object_columns_df['BsmtCond'].map(bin_map)
 object_columns_df['BsmtQual'] = object_columns_df['BsmtQual'].map(bin_map)
 object_columns_df['HeatingQC'] = object_columns_df['HeatingQC'].map(bin_map)
 object_columns_df['KitchenQual'] = object_columns_df['KitchenQual'].map(bin_map)
 object_columns_df['FireplaceQu'] = object_columns_df['FireplaceQu'].map(bin_map)
 object_columns_df['GarageQual'] = object_columns_df['GarageQual'].map(bin_map)
 object_columns_df['GarageCond'] = object_columns_df['GarageCond'].map(bin_map)
 object_columns_df['CentralAir'] = object_columns_df['CentralAir'].map(bin_map)
 object_columns_df['LotShape'] = object_columns_df['LotShape'].map(bin_map)
 object_columns_df['BsmtExposure'] = object_columns_df['BsmtExposure'].map(bin_map)
 object_columns_df['BsmtFinType1'] = object_columns_df['BsmtFinType1'].map(bin_map)
 object_columns_df['BsmtFinType2'] = object_columns_df['BsmtFinType2'].map(bin_map)
 PavedDrive = {"N" : 0, "P" : 1, "Y" : 2}
object_columns_df['PavedDrive'] = object_columns_df['PavedDrive'].map(PavedDrive)
<categorify>
f_drop_num = [
'BedroomAbvGr','OverallCond','MSSubClass','KitchenAbvGr'
]
rest_object_columns = object_columns_df.select_dtypes(include=['object'])
 object_columns_df = pd.get_dummies(object_columns_df, columns=rest_object_columns.columns)
<concatenate>
print(*[c for c in train.select_dtypes(include=['object'] ).columns if c in basement], sep=", ")
 basement_categorical = [c for c in train.select_dtypes(include=['object'] ).columns if c in basement]
df_final = pd.concat([object_columns_df, numerical_columns_df], axis=1,sort=False)
 df_final.head()<prepare_x_and_y>
f_categorical = [f for f in all_data.columns if all_data.dtypes[f] == 'object']
for c in f_categorical:
all_data[c] = all_data[c].astype('category')
 if all_data[c].isnull().any() :
all_data[c] = all_data[c].cat.add_categories(['MISSING'])
 all_data[c] = all_data[c].fillna('MISSING' )
target= df_train['SalePrice']
df_train = df_train.drop(['SalePrice'],axis=1 )<drop_column>
def make_MISSING(data):
f_categorical = [f for f in data.columns if data.dtypes[f] == 'object']
for c in f_categorical:
data[c] = data[c].astype('category')
 if data[c].isnull().any() :
data[c] = data[c].cat.add_categories(['MISSING'])
 data[c] = data[c].fillna('MISSING' )
df_test = df_test.drop(['SalePrice'],axis=1)
 df_test.head()<split>
BsmtQual_map = {"Ex":3, "Gd":2, "TA":2, "Fa":1, "MISSING":1}
BsmtFinType1_map = {"GLQ":2, "ALQ":2, "Unf":2,"Rec":1, "LwQ":1, "BLQ":1, "MISSING":1}
BsmtCond_map = {"Gd":3, "TA":3, "MISSING":2, "Fa":2, "Po":1}
BsmtFinType2_map = {"Unf":4, "ALQ":4, "MISSING":3, "Rec":3, "BLQ":2, "LwQ":2, "GLQ":1 }
all_data["BsmtQual"] = all_data.BsmtQual.replace(BsmtQual_map)
 all_data["BsmtFinType1"] = all_data.BsmtFinType1.replace(BsmtFinType1_map)
 all_data["BsmtCond"] = all_data.BsmtCond.replace(BsmtCond_map)
 all_data["BsmtFinType2"] = all_data.BsmtFinType2.replace(BsmtFinType2_map )
x_train,x_test,y_train,y_test = train_test_split(df_train,target,test_size=0.2,random_state=0 )<choose_model_class>
pd.options.display.max_rows = 1000
pd.options.display.max_columns = 25
xgb =XGBRegressor(booster='gbtree', colsample_bylevel=1,
colsample_bynode=1, colsample_bytree=0.6, gamma=0,
importance_type='gain', learning_rate=0.01, max_delta_step=0,
max_depth=4, min_child_weight=1.5, n_estimators=2400,
n_jobs=1, nthread=None, objective='reg:linear',
reg_alpha=0.6, reg_lambda=0.6, scale_pos_weight=1,
silent=None, subsample=0.8, verbosity=1)
 lgbm = LGBMRegressor(objective='regression',
num_leaves=4,
learning_rate=0.01,
n_estimators=12000,
max_bin=200,
bagging_fraction=0.75,
bagging_freq=5,
bagging_seed=7,
feature_fraction=0.4, 
)
<train_model>
ExterQual_map = {"Gd":2, "TA":2, "Ex":3, "Fa":1}
FireplaceQu_map = {"MISSING":2, "TA":3, "Gd":3,"Fa":1, "Ex":4, "Po":1}
KitchenQual_map = {"Gd":1, "TA":2, "MISSING":5, "Fa":4, "Ex":3}
MSZoning_map = {"RL":1, "RM":1, "C（all）":2, "FV":3, "RH":3, "MISSING":4}
GarageType_map = {"Attchd":1, "Detchd":1, "BuiltIn":2, "CarPort":3, "MISSING":1, "Basment":4, "2Types":4}
Neighborhood_map = {}
BldgType_map = {"1Fam":1, "2fmCon":2, "Duplex":2, "TwnhsE":1, "Twnhs":2}
GarageFinish_map = {"RFn":1, "Unf":2, "Fin":1,"MISSING":2}
Foundation_map = {"PConc":1, "CBlock":1, "BrkTil":2, "Wood":3, "Slab":2, "Stone":3}
HouseStyle_map = {"2Story":1, "1Story":1, "1.5Fin":2,"1.5Unf":3, "SFoyer":4, "SLvl":2, "2.5Unf":4, "2.5Fin":5}
CentralAir_map = {"Y":1, "N":2}
HeatingQC_map = {"Ex":1, "Gd":1, "TA":1,"Fa":2, "Po":3}
GarageCond_map = {"TA":1, "Fa":2, "MISSING":1,"Gd":2, "Po":2, "Ex":3}
GarageQual_map = {"TA":1, "Fa":2, "Gd":3,"MISSING":1, "Ex":4, "Po":5}
Exterior1st_map = {}
Exterior2nd_map = {}
PavedDrive_map = {"Y":1, "N":2, "P":2}
LandContour_map = {"Lvl":1, "Bnk":2, "Low":3,"HLS":1}
MasVnrType_map = {"BrkFace":1, "None":1, "Stone":3,"BrkCmn":2, "MISSING":3}
Functional_map = {"Typ":1, "Min1":4, "Maj1":2,"Min2":1, "Mod":1, "Maj2":2, "Sev":3, "MISSING":2}
Fence_map = {"MISSING":1, "MnPrv":1, "GdWo":2,"GdPrv":2, "MnWw":3}
all_data["ExterQual"] = all_data.ExterQual.replace(ExterQual_map)
 all_data["FireplaceQu"] = all_data.FireplaceQu.replace(FireplaceQu_map)
 all_data["KitchenQual"] = all_data.KitchenQual.replace(KitchenQual_map)
 all_data["MSZoning"] = all_data.MSZoning.replace(MSZoning_map)
 all_data["GarageType"] = all_data.GarageType.replace(GarageType_map)
 all_data["BldgType"] = all_data.BldgType.replace(BldgType_map)
 all_data["GarageFinish"] = all_data.GarageFinish.replace(GarageFinish_map)
 all_data["Foundation"] = all_data.Foundation.replace(Foundation_map)
 all_data["HouseStyle"] = all_data.HouseStyle.replace(HouseStyle_map)
 all_data["CentralAir"] = all_data.CentralAir.replace(CentralAir_map)
 all_data["HeatingQC"] = all_data.HeatingQC.replace(HeatingQC_map)
 all_data["GarageCond"] = all_data.GarageCond.replace(GarageCond_map)
 all_data["GarageQual"] = all_data.GarageQual.replace(GarageQual_map)
 all_data["PavedDrive"] = all_data.PavedDrive.replace(PavedDrive_map)
 all_data["LandContour"] = all_data.LandContour.replace(LandContour_map)
 all_data["MasVnrType"] = all_data.MasVnrType.replace(MasVnrType_map)
 all_data["Functional"] = all_data.Functional.replace(Functional_map)
 all_data["Fence"] = all_data.Fence.replace(Fence_map )
xgb.fit(x_train, y_train)
 lgbm.fit(x_train, y_train,eval_metric='rmse')
<predict_on_test>
Neighborhood_map = {
'CollgCr':6, 'Veenker':4, 'Crawfor':6, 'NoRidge':1, 'Mitchel':5,
'Blmngtn':5, 'Blueste':5, 'BrDale':1, 'ClearCr':7, 'Edwards':2,
'Gilbert':5, 'IDOTRR':5, 'MeadowV':5, 'NAmes':5, 'NPkVill':5,
'NWAmes':0, 'NridgHt':4, 'OldTown':0, 'SWISU':5, 'Sawyer':0,
'SawyerW':4, 'Somerst':0, 'StoneBr':3, 'Timber':4, 'BrkSide':5,
}
all_data["Neighborhood"] = all_data.Neighborhood.replace(Neighborhood_map )
predict1 = xgb.predict(x_test)
 predict = lgbm.predict(x_test)
<compute_test_metric>
Exterior1st_map = {
'AsbShng':7, 'AsphShn':6, 'BrkComm':8, 'BrkFace':7, 'CBlock':6,
'CemntBd':5, 'HdBoard':3, 'ImStucc':0, 'MetalSd':4, 'Plywood':3,
'Stone':2, 'Stucco':4, 'VinylSd':3, 'Wd Sdng':3, 'WdShing':3,
'MISSING':1,
}
all_data["Exterior1st"] = all_data.Exterior1st.replace(Exterior1st_map )
print('Root Mean Square Error test = ' + str(math.sqrt(metrics.mean_squared_error(y_test, predict1))))
print('Root Mean Square Error test = ' + str(math.sqrt(metrics.mean_squared_error(y_test, predict)))) 
<train_model>
Exterior2nd_map = {
'AsbShng':7, 'AsphShn':8, 'Brk Cmm':7, 'BrkFace':7, 'CBlock':6,
'CmentBd':5, 'HdBoard':4, 'ImStucc':5, 'MetalSd':4, 'Other':0, 'Plywood':4,
'Stone':3, 'Stucco':3, 'VinylSd':2, 'Wd Sdng':2, 'Wd Shng':2,
'MISSING':1,
}
all_data["Exterior2nd"] = all_data.Exterior2nd.replace(Exterior2nd_map )
xgb.fit(df_train, target)
 lgbm.fit(df_train, target,eval_metric='rmse')
<predict_on_test>
MISSING_KitchenQual = all_data.loc[all_data['KitchenQual'] == 5]
MISSING_Exterior1st_0 = all_data.loc[all_data['Exterior1st'] == 0]
MISSING_Exterior1st_1 = all_data.loc[all_data['Exterior1st'] == 1]
MISSING_Exterior2nd_0 = all_data.loc[all_data['Exterior2nd'] == 0]
MISSING_Exterior2nd_1 = all_data.loc[all_data['Exterior2nd'] == 1]
MISSING_KitchenQual
MISSING_Exterior1st_0
MISSING_Exterior1st_1
MISSING_Exterior2nd_0
MISSING_Exterior2nd_1
predict4 = lgbm.predict(df_test)
 predict3 = xgb.predict(df_test)
 predict_y =(predict3*0.575 + predict4 * 0.425)
<save_to_csv>
all_data.drop([95, 1187, 691, 595, 691], inplace=True)
 test['Id'].drop([95, 1187, 691, 595, 691], inplace=True )
submission = pd.DataFrame({
"Id": test["Id"],
"SalePrice": predict_y
})
 submission.to_csv('submission.csv', index=False)
 print('Submission completed')
<load_from_csv>
train['SalePrice'] = np.log1p(train['SalePrice'])
 numeric_feats = train.dtypes[train.dtypes != "object"].index
skewed_feats = train[numeric_feats].apply(lambda x: skew(x.dropna()))
skewed_feats = skewed_feats[skewed_feats > 0.75]
skewed_feats = skewed_feats.index
all_data[skewed_feats] = np.log1p(all_data[skewed_feats] )
train = pd.read_csv('.. /input/ames-housing-dataset/AmesHousing.csv')
 train.drop(['PID'], axis=1, inplace=True)
 origin = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 train.columns = origin.columns
test = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv')
 submission = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/sample_submission.csv')
 print('Train:{} Test:{}'.format(train.shape,test.shape))<drop_column>
for feature in all_data.select_dtypes(exclude=['object', 'category'] ).columns:
all_data[feature].fillna(all_data[feature].median() , inplace = True)
missing = test.isnull().sum()
missing = missing[missing>0]
train.drop(missing.index, axis=1, inplace=True)
 train.drop(['Electrical'], axis=1, inplace=True)
 test.dropna(axis=1, inplace=True)
 test.drop(['Electrical'], axis=1, inplace=True )<feature_engineering>
all_data.drop('SalePrice', axis=1, inplace=True)
l_test = tqdm(range(0, len(test)) , desc='Matching')
 for i in l_test:
for j in range(0, len(train)) :
for k in range(1, len(test.columns)) :
if test.iloc[i,k] == train.iloc[j,k]:
continue
else:
break
else:
submission.iloc[i, 1] = train.iloc[j, -1]
break
l_test.close()<save_to_csv>
all_data = pd.get_dummies(all_data)
 print(all_data.shape )
submission.to_csv('result-with-best.csv', index=False )<import_modules>
X_train = all_data[:train.shape[0]]
X_test = all_data[train.shape[0]:]
y = train.SalePrice
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import LabelEncoder, OneHotEncoder
from sklearn.compose import ColumnTransformer
from sklearn.linear_model import LinearRegression
from sklearn.linear_model import Ridge
from sklearn.linear_model import Lasso
from sklearn.ensemble import RandomForestRegressor
from sklearn.impute import SimpleImputer
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import train_test_split
from sklearn import metrics
from sklearn.metrics import mean_squared_log_error
from sklearn.metrics import mean_squared_error
from xgboost import XGBRegressor
import math<compute_test_metric>
train_x, test_x, train_y, test_y = train_test_split(X_train, y, test_size=0.3, random_state=0 )
def root_mean_squared_log_error(y_validations, y_predicted):
if len(y_predicted)!= len(y_validations):
return 'error: mismatch in number of data points between x and y'
y_predict_modified = [math.log(i)for i in y_predicted]
y_validations_modified = [math.log(i)for i in y_validations]
return mean_squared_error(y_validations_modified, y_predict_modified, squared=False )<load_from_csv>
ENet = make_pipeline(RobustScaler() , ElasticNet(alpha=0.0005, l1_ratio=0.5, random_state=3))
df_train = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 df_test = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv' )<define_variables>
ENet.fit(train_x, train_y)
 ENet.score(test_x,test_y)*100
year_feature = [feature for feature in numeric_features if 'Yr' in feature or 'Year' in feature]
year_feature<sort_values>
ENet.fit(X_train.values, y)
 ENet.score(test_x,test_y)*100
top_correlations = df_train.corr()
top_feature_columns = top_correlations['SalePrice'][top_correlations['SalePrice'].values > 0.2].index.values
top_feature_columns<sort_values>
def predict_cv(model, train_x, train_y, test_x):
preds = []
preds_test = []
va_indexes = []
kf = KFold(n_splits=4, shuffle=True, random_state=6785)
 for i,(tr_idx, va_idx)in enumerate(kf.split(train_x)) :
tr_x, va_x = train_x.iloc[tr_idx], train_x.iloc[va_idx]
tr_y, va_y = train_y.iloc[tr_idx], train_y.iloc[va_idx]
model.fit(tr_x, tr_y)
 tr_pred = model.predict(tr_x)
 pred = model.predict(va_x)
 preds.append(pred)
 pred_test = model.predict(test_x)
 preds_test.append(pred_test)
 va_indexes.append(va_idx)
 print(' score Train : {:.6f}'.format(np.sqrt(mean_squared_error(tr_y, tr_pred))),
' score Valid : {:.6f}'.format(np.sqrt(mean_squared_error(va_y, pred))),
' Correct answer rate Train: {:.6f}'.format(model.score(tr_x, tr_y)) ,
' Correct answer rate Valid: {:.6f}'.format(model.score(va_x, va_y)) , 
)
 va_indexes = np.concatenate(va_indexes)
 preds = np.concatenate(preds, axis=0)
 order = np.argsort(va_indexes)
 pred_train = pd.DataFrame(preds[order])
 preds_test = pd.DataFrame(np.mean(preds_test, axis=0))
print('Score : {:.6f}'.format(np.sqrt(mean_squared_error(train_y, pred_train))))
return pred_train, preds_test, model
total = df_train.isnull().sum().sort_values(ascending=False)
 percent =(df_train.isnull().sum() / df_train.isnull().count() ).sort_values(ascending=False)
 missing_data = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
 missing_data.head(20 )<compute_test_metric>
predict_cv(ENet, train_x, train_y, test_x )
def loss_func(y_predicted, y_actual):
squared_error =(y_predicted - y_actual)** 2
sum_squared_error = squared_error.sum()
size = len(y_actual)
 return 1/(2*size)* sum_squared_error<prepare_x_and_y>
ENet.fit(X_train.values, y)
 ENet.score(test_x,test_y)*100
x = df_train['GrLivArea']
y = df_train['SalePrice']
x =(x - x.mean())/ x.std()
x = np.c_[np.ones(x.shape[0]), x]
x.shape<prepare_x_and_y>
ENet_pred = np.expm1(ENet.predict(X_test.values))
<categorify><EOS> 
sub = pd.DataFrame()
sub['Id'] = test['Id']
sub['SalePrice'] = ENet_pred
sub.to_csv('submission.csv',index=False )