Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<filter>
warnings.filterwarnings("ignore")
data_df[(data_df['MasVnrType'].isnull())|(data_df['MasVnrArea'].isnull())].loc[:, ['MasVnrType', 'MasVnrArea']]<feature_engineering>
from scipy import stats
from scipy.special import boxcox1p
from scipy.stats import skew
from sklearn.metrics import r2_score, mean_squared_log_error
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import StratifiedKFold
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import RobustScaler
from sklearn.model_selection import KFold
from sklearn.pipeline import make_pipeline
from sklearn.kernel_ridge import KernelRidge
len(data_df[data_df['MasVnrArea'] == 0])/len(data_df['MasVnrArea'] )<data_type_conversions>
plt.style.use('seaborn')
 plt.rcParams['figure.figsize'] = [15, 5]
plt.rcParams['axes.labelsize'] = 15
plt.rcParams['axes.titlesize'] = 20
pd.set_option('display.max_columns', 500)
 pd.set_option('display.max_rows', 200 )
train_df['MasVnrArea'].fillna(0, inplace=True )<data_type_conversions>
df_train = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv')
 df_test = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv')
 df_all = pd.concat(( df_train,df_test),ignore_index=True )
train_df['MasVnrType'].fillna('None', inplace=True )<count_missing_values>
train_len = len(df_train)
 df_all["Set"] = ["Train"]*train_len + ["Test"]*(len(df_all)- train_len )
print('Number of missing values in MasVnrType ', test_df['MasVnrType'].isnull().sum() , 'and MasVnArea ', test_df['MasVnrArea'].isnull().sum() )<filter>
print("Skewness of target: ",skew(df_all["SalePrice"].dropna()))
test_df[(test_df['MasVnrType'].isnull())| 
(test_df['MasVnrArea'].isnull())].loc[:, ['MasVnrType', 'MasVnrArea']]<count_values>
df_all["SalePrice_log"] = df_all["SalePrice"].apply(np.log1p )
test_df['MasVnrType'].value_counts()<feature_engineering>
nan_count = df_all.isnull().sum()
nan_count = nan_count[nan_count>0].sort_values(ascending=False)
 nan_count
test_df.at[1150, 'MasVnrType'] = 'BrkFace'
test_df['MasVnrType'].fillna('None', inplace=True)
 test_df['MasVnrArea'].fillna(test_df['MasVnrArea'].median() , inplace=True )<count_missing_values>
nan_is_none_cols = ["PoolQC", "MiscFeature", "Alley", "Fence", "GarageQual", \
"GarageCond", "GarageType","BsmtCond","BsmtExposure","BsmtQual",\
"BsmtFinType2","BsmtFinType1","MasVnrType","FireplaceQu","GarageFinish"]
for c in nan_is_none_cols:
df_all[c] = df_all[c].fillna("None" )
print('Number of missing values = ', data_df['Electrical'].isnull().sum() )<count_values>
df_all.loc[df_all["MasVnrArea"].isnull() ,"MasVnrArea"] = 0
data_df['Electrical'].value_counts()<data_type_conversions>
df_all.corr() ["LotFrontage"].abs().sort_values(ascending=False ).head()
train_df['Electrical'].fillna('Sbrkr', inplace=True )<count_missing_values>
data = df_all.drop(["SalePrice","Set"], axis=1)
 for c in data.select_dtypes(exclude=["float","int"]):
data[c] = data[c].factorize() [0]
print('Number of missing values = ', test_df['Electrical'].isnull().sum() )<count_missing_values>
data.corr().abs() ["LotFrontage"].sort_values(ascending=False ).head(10 )
train_df.isnull().sum().sum()<filter>
model = Ridge()
data=data.dropna()
x_ = data.drop("LotFrontage",axis=1)
 y_ = data["LotFrontage"]
model.fit(x_, y_)
 data["interpolated_LotFrontage"] = model.predict(x_)
 data.corr().abs() ["LotFrontage"].sort_values(ascending=False ).head()
test_df.columns[np.where(test_df.isnull().sum() != 0)]<data_type_conversions>
df_all["LotFrontage"] = df_all.groupby("Neighborhood")["LotFrontage"].transform(
 lambda x: x.fillna(x.median()))
test_df['MSZoning'].fillna('RL', inplace=True)
 test_df['Utilities'].fillna('AllPub', inplace=True)
 test_df['Exterior1st'].fillna('VinylSd', inplace=True)
 test_df['Exterior2nd'].fillna('VinylSd', inplace=True)
 test_df['BsmtFullBath'].fillna(0, inplace=True)
 test_df['BsmtHalfBath'].fillna(0, inplace=True)
 test_df['KitchenQual'].fillna('TA', inplace=True)
 test_df['Functional'].fillna('Typ', inplace=True)
 test_df['GarageCars'].fillna(2, inplace=True)
 test_df['SaleType'].fillna('WD', inplace=True )<count_missing_values>
df_all.loc[df_all["TotalBsmtSF"].isnull() ,["BsmtCond","BsmtUnfSF","BsmtFinSF1","BsmtFinSF2","BsmtHalfBath","BsmtFullBath","Set"]]
test_df.isnull().sum().sum()<drop_column>
df_all.loc[df_all["TotalBsmtSF"].isnull() ,["BsmtUnfSF","BsmtFinSF1","BsmtFinSF2","BsmtHalfBath","BsmtFullBath"]] = 0
df_all["TotalBsmtSF"] = df_all["TotalBsmtSF"].fillna(0 )
train_df.drop(columns=['Id', 'SalePrice'], inplace=True)
 test_df.drop(columns=['Id'], inplace=True )<define_variables>
df_all["Functional"].value_counts()
print('Current list of outliers ', list(outliers))<drop_column>
df_all["Functional"] = df_all["Functional"].fillna(df_all["Functional"].mode() [0] )
train_df.drop(index=list(outliers), inplace=True )<drop_column>
df_all.loc[df_all["MSZoning"].isnull() ,:]
data_df.drop(index=list(outliers), inplace=True )<categorify>
df_all["MSZoning"].value_counts()
dummies = pd.get_dummies(pd.concat(( train_df, test_df), axis=0))<prepare_x_and_y>
df_all["MSZoning"] = df_all["MSZoning"].fillna(df_all["MSZoning"].mode() [0] )
X = dummies.iloc[:train_df.shape[0]]
X_test = dummies.iloc[train_df.shape[0]:]<prepare_x_and_y>
df_all.loc[df_all["Utilities"].isnull() ,:]
y = np.log(data_df['SalePrice'] + 1 )<compute_train_metric>
df_all["Utilities"].value_counts()
GBR3 = GradientBoostingRegressor(learning_rate= 0.1, n_estimators=150, max_depth=4)
 cv_score = cross_val_score(GBR3, X, y, cv=5, scoring='neg_mean_squared_error')
 print('Cross validation scores for GBR model:', np.sqrt(-cv_score ).mean() )<compute_train_metric>
df_all["Utilities"] = df_all["Utilities"].fillna(df_all["Utilities"].mode() [0] )
RF3 = RandomForestRegressor(n_estimators=100, max_features=20, random_state=0)
 cv_score2 = cross_val_score(RF3, X, y, cv=5, scoring='neg_mean_squared_error')
 print('Cross validation scores for GBR model:', np.sqrt(-cv_score2 ).mean() )<save_to_csv>
df_all.loc[df_all["BsmtHalfBath"].isnull() ,["BsmtFullBath","BsmtCond","BsmtQual","Set"]]
GBR3.fit(X, y)
 y_pred = np.exp(GBR3.predict(X_test)) - 1
answer = pd.DataFrame(data=y_pred, columns=['SalePrice'])
 answer.insert(loc=0, column='Id', value=test_data_df['Id'])
 answer.to_csv('submission.csv', index=False )<import_modules>
df_all["BsmtHalfBath"] = df_all["BsmtHalfBath"].fillna(0)
 df_all["BsmtFullBath"] = df_all["BsmtFullBath"].fillna(0 )
print('*'*50)
 print('Pandas Version : ', pd.__version__)
 print('Numpy Version : ', np.__version__)
 print('Matplotlib Version: ', mpl.__version__)
 print('Seaborn Version : ', sns.__version__)
 print('SKLearn Version : ', sk.__version__)
 print('*'*50 )<set_options>
df_all.loc[df_all["GarageArea"].isnull() ,["GarageArea","GarageCars","GarageType","GarageFinish","Set","GarageYrBlt","GarageQual","GarageCond"]]
sns.set_style('whitegrid')
 pd.options.display.max_rows = 100
pd.options.display.max_columns = 100
seed = 42
v_size = 0.33
num_folds = 10
scoring = 'accuracy'
np.random.seed(seed )<load_from_csv>
df_all.loc[df_all["GarageArea"].isnull() ,"GarageType"] = "None"
df_all.loc[df_all["GarageArea"].isnull() ,["GarageArea","GarageCars","GarageYrBlt"]] = 0
df_all["GarageYrBlt"] = df_all["GarageYrBlt"].fillna(0 )
path_train = '.. /input/train.csv'
path_test = '.. /input/test.csv'
df_train = pd.read_csv(path_train)
 df_test = pd.read_csv(path_test )<train_model>
df_all.loc[df_all["Exterior1st"].isnull() ,["Exterior1st","YearBuilt","OverallQual","RoofStyle","Set"]]
print('Datasets shapes: ')
 print("Training shape: ", df_train.shape)
 print("Test shape : ", df_test.shape )<count_missing_values>
df_all["Exterior1st"].value_counts() / len(df_all )
missingData(df_train )<count_missing_values>
df_all["Exterior2nd"].value_counts() / len(df_all )
missingData(df_test )<drop_column>
df_all["Exterior1st"] = df_all["Exterior1st"].fillna(df_all["Exterior1st"].mode() [0])
 df_all["Exterior2nd"] = df_all["Exterior2nd"].fillna(df_all["Exterior2nd"].mode() [0] )
df_train.drop("Cabin", axis=1, inplace = True)
 df_test.drop("Cabin", axis=1, inplace = True )<data_type_conversions>
df_all.loc[df_all["KitchenQual"].isnull() ,["KitchenAbvGr","Set","YearBuilt","OverallQual"]]
df_train["Age"].fillna(df_train["Age"].median() , inplace = True)
 df_test["Age"].fillna(df_test["Age"].median() , inplace = True )<data_type_conversions>
df_all["KitchenQual"].value_counts()
df_train['Embarked'] = df_train['Embarked'].fillna(df_train['Embarked'].mode() [0] )<count_missing_values>
df_all["KitchenQual"] = df_all["KitchenQual"].fillna("Gd" )
def displayNanValues() :
print("Check the NaN value in train data")
 print(df_train.isnull().sum())
 print("---"*30)
 print("Check the NaN value in test data")
 print(df_test.isnull().sum())
 print("---"*30 )<define_variables>
df_all["SaleType"] = df_all["SaleType"].fillna(df_all["SaleType"].mode() [0] )
all_data = [df_train, df_test]<feature_engineering>
df_all.loc[df_all["Electrical"].isnull() ,["Set","YearBuilt"]]
def featureExtraction(all_data):
for dataset in all_data:
dataset['FamilySize'] = dataset['SibSp'] + dataset['Parch'] + 1
for dataset in all_data:
dataset['Age_bin'] = pd.cut(dataset['Age'], bins=[0,18,60,120], labels=['Children','Adult','Elder'])
 for dataset in all_data:
dataset['Title'] = dataset.Name.str.extract('([A-Za-z]+)\.', expand=False)
 for dataset in all_data:
dataset['Title'] = dataset['Title'].replace(['Lady', 'Countess','Capt', 'Col',
'Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')
 dataset['Title'] = dataset['Title'].replace('Mlle', 'Miss')
 dataset['Title'] = dataset['Title'].replace('Ms', 'Miss')
 dataset['Title'] = dataset['Title'].replace('Mme', 'Mrs' )<feature_engineering>
df_all["Electrical"] = df_all["Electrical"].fillna("SBrkr" )
featureExtraction(all_data )<create_dataframe>
df_all["Electrical"].value_counts()
dfTrain = df_train.copy()
dfTest = df_test.copy()<categorify>
nan_count =(df_all.isnull().sum())
 nan_count[nan_count>0].sort_values(ascending=False )
traindf = pd.get_dummies(dfTrain, columns = ["Pclass","Title",'FamilySize',"Sex","Age_bin","Embarked"],
prefix=["Pclass","Title",'FamilySize',"Sex","Age_type","Em_type"])
 testdf = pd.get_dummies(dfTest, columns = ["Pclass","Title",'FamilySize',"Sex","Age_bin","Embarked"],
prefix=["Pclass","Title",'FamilySize',"Sex","Age_type","Em_type"] )<define_variables>
col_cond = {
"hasPool": "PoolQC",
"hasMiscFeature": "MiscFeature",
"hasAlley": "Alley",
"hasFence": "Fence",
"hasGarage" : "GarageQual",
"hasBasement" : "BsmtCond",
"hasVeneer" : "MasVnrType"
}
for col in col_cond:
df_all[col] = 1
df_all.loc[df_all[col_cond[col]]=="None",col] = 0
df_all["has2ndFloor"] = 1
df_all.loc[df_all["2ndFlrSF"]==0,"has2ndFloor"] = 0
df_all["hasCondition"]= 1
df_all.loc[(df_all["Condition1"]=="Norm")&(df_all["Condition2"]=="Norm"),"hasCondition"] =0
df_all["wasRemodeled"] = 0
df_all.loc[df_all["YearRemodAdd"]>df_all["YearBuilt"],"wasRemodeled"] = 1
df_all["houseAge"] = df_all["YrSold"] - df_all["YearBuilt"]
df_all["garageAge"] = df_all["YrSold"] - df_all["GarageYrBlt"]
df_all["garageAge"] = df_all["garageAge"].apply(lambda age: 0 if age>1000 else age)
 df_all["hasLowQualFinish"] = 0
df_all.loc[df_all["LowQualFinSF"]>0,"hasLowQualFinish"] = 1
df_all["totalFullBath"] = df_all["BsmtFullBath"] + df_all["FullBath"]
df_all["totalHalfBath"] = df_all["BsmtHalfBath"] + df_all["HalfBath"]
df_all["totalBath"] = df_all["totalHalfBath"] + df_all["totalFullBath"]
df_all["TotRmsAbvGrd+baths"] = df_all["TotRmsAbvGrd"] + df_all["FullBath"] + df_all["HalfBath"]
df_all["totalHouseSF"] = df_all['1stFlrSF'] + df_all['2ndFlrSF'] + df_all["TotalBsmtSF"]
df_all["hasDeck"] = 0
df_all.loc[df_all["WoodDeckSF"]>0,"hasDeck"] = 1
df_all["hasOpenPorch"] = 0
df_all.loc[df_all["OpenPorchSF"]>0,"hasOpenPorch"] = 1
df_all["hasEnclosedPorch"] = 0
df_all.loc[df_all["EnclosedPorch"]>0,"hasEnclosedPorch"] = 1
df_all["has3SsnPorch"] = 0
df_all.loc[df_all["3SsnPorch"]>0,"has3SsnPorch"] = 1
df_all["hasScreenPorch"] = 0
df_all.loc[df_all["ScreenPorch"]>0,"hasScreenPorch"] = 1
df_all["DeckPorchCount"] = df_all["hasDeck"] + df_all["hasOpenPorch"] + \
df_all["hasEnclosedPorch"] + df_all["has3SsnPorch"] + df_all["hasScreenPorch"]
df_all["totalDeckPorchSF"] = df_all["WoodDeckSF"] + df_all["OpenPorchSF"] +\
df_all["EnclosedPorch"] + df_all["3SsnPorch"] + df_all["ScreenPorch"]
df_all["landSF"] = df_all["LotArea"] - df_all["1stFlrSF"]
df_all["SalePriceSF"] = df_all["SalePrice"] /(df_all["landSF"] + df_all["totalHouseSF"])
 df_all["hasRegularShape"]= 0
df_all.loc[df_all["LotShape"]=="Reg","hasRegularShape"] = 1
df_all["Generation"] = pd.cut(df_all["YearBuilt"],bins=[0,1949,1985,2010], right=True,\
labels=[0,1,2])
 df_all["Bsmt>GrLivArea"] = 0
df_all.loc[df_all["TotalBsmtSF"]>df_all["GrLivArea"], "Bsmt>GrLivArea" ] = 1
df_all["Bsmt>1stFlrSF"] = 0
df_all.loc[df_all["TotalBsmtSF"]>df_all["1stFlrSF"], "Bsmt>1stFlrSF" ] = 1
df_all["1stFlrSF>2ndFlrSF"] = 0
df_all.loc[df_all["1stFlrSF"]>df_all["2ndFlrSF"], "1stFlrSF>2ndFlrSF" ] = 1
allData = [traindf, testdf]<drop_column>
outlier_lotfrontage = df_all.loc[df_all["LotFrontage"]>300].index.tolist()
df_all.iloc[outlier_lotfrontage]
for dataset in allData:
drop_column = ["Age","Fare","Name","Ticket","SibSp","Parch"]
dataset.drop(drop_column, axis=1, inplace = True)
 traindf.drop(["PassengerId"], axis=1, inplace = True )<randomize_order>
outlier_lotarea = df_all.loc[df_all["LotArea"]>100000].index.tolist()
df_all.iloc[outlier_lotarea]
df_data_shuffled = traindf.iloc[np.random.permutation(len(traindf)) ]<data_type_conversions>
df_all["Street"].value_counts()
array = df_data_shuffled.values
features = array[:,1:].astype(float)
 targeted = array[:,0].astype(float )<split>
df_all["Alley"].value_counts()
X_train,X_test,y_train,y_test = train_test_split(features,targeted,test_size=v_size,random_state=seed)
 print('Data shapes: ')
 print("X_train shape: ", X_train.shape)
 print("X_test shape : ", X_test.shape)
 print("y_train shape: ", y_train.shape)
 print("y_test shape : ", y_test.shape )<train_on_grid>
df_all["LotShape"].value_counts()
lr_clf = LogisticRegression()
lr_param_grid = {'solver' : ['liblinear', 'lbfgs'],'C': [0.001, 0.01, 0.1, 1, 10, 100, 1000] }
best_lr = algoGridTune(lr_clf, lr_param_grid, X_train, y_train,num_folds, scoring, seed )<train_on_grid>
df_all["Condition1"].value_counts() / len(df_all )
knn_clf = KNeighborsClassifier()
knn_param_grid = {'n_neighbors':[3,5,7,9,11],
'leaf_size':[1,2,3,5],
'weights':['uniform', 'distance'],
'algorithm':['auto', 'ball_tree','kd_tree','brute']
}
best_knn = algoGridTune(knn_clf, knn_param_grid, X_train, y_train, num_folds, scoring, seed )<train_on_grid>
df_all["Condition2"].value_counts() / len(df_all )
dt_clf = DecisionTreeClassifier()
dt_param_grid = {'max_depth' : [3,4,5,6,7,8,9,10],
'max_features': ['sqrt', 'log2'],
'min_samples_split': [3,5,7,9,11],
'min_samples_leaf':[1,3,5,7,9,11]
}
best_dt = algoGridTune(dt_clf, dt_param_grid, X_train, y_train,num_folds, scoring, seed )<train_on_grid>
df_all["BldgType"].value_counts()
svc_clf = SVC()
svc_param_grid = [{"kernel": ["rbf"],
"gamma": [10 ,1, 0.1, 1e-2, 1e-3],
"C": [0.1,1,10],
"random_state" : [seed]},
{"kernel": ["linear"], "C": [0.1,1,10,100]}
]
best_SVC = algoGridTune(svc_clf, svc_param_grid, X_train, y_train, num_folds, scoring, seed )<choose_model_class>
df_all["HouseStyle"].value_counts()
DTC = DecisionTreeClassifier()
ABC_clf = AdaBoostClassifier(DTC, random_state=seed)
 ABC_param_grid = {"base_estimator__criterion" : ["gini"],
"base_estimator__splitter" : ["best"],
"algorithm" : ["SAMME"],
"n_estimators" :[30, 100],
"learning_rate": [0.0001, 0.001, 0.01, 0.1, 0.2, 0.3,1.5]}
best_ABC = algoGridTune(ABC_clf, ABC_param_grid, X_train, y_train, num_folds, scoring, seed )<train_on_grid>
print("Square feet saleprice std withing OverallQual categories:")
 df_all.groupby("OverallQual")["SalePriceSF"].std()
ETC_clf = ExtraTreesClassifier()
ETC_param_grid = {"max_depth": [30],
"max_features": ['sqrt'],
"min_samples_split": [2, 3, 5],
"min_samples_leaf": [1, 3, 5],
"bootstrap": [True],
"n_estimators" :[300],
"criterion": ["gini"]}
best_ETC = algoGridTune(ETC_clf, ETC_param_grid, X_train, y_train, num_folds, scoring, seed )<choose_model_class>
outlier_overallqual = df_all.loc[(df_all["OverallQual"]==10)&\ 
(df_all["SalePriceSF"]<10),:].index.tolist()
df_all.iloc[outlier_overallqual]
GBC_clf = GradientBoostingClassifier()
GBC_param_grid = {'loss' : ["deviance"],
'n_estimators' : [100, 200, 300],
'learning_rate': [0.1, 0.05, 0.01],
'max_depth': [3, 5, 7],
'min_samples_leaf': [1, 5, 9],
'min_samples_split': [2, 6, 10],
'max_features': ['sqrt', 'log2']
}
best_GBC = algoGridTune(GBC_clf, GBC_param_grid, X_train, y_train, num_folds, scoring, seed )<train_on_grid>
df_all["RoofStyle"].value_counts()
RFC_clf = RandomForestClassifier()
RFC_param_grid = {"max_depth": [None],
"max_features": [1, 3, 10],
"min_samples_split": [2, 3, 10],
"min_samples_leaf": [1, 3, 10],
"bootstrap": [True, False],
"n_estimators" :[100,300],
"criterion": ["gini"]}
best_RFC = algoGridTune(RFC_clf, RFC_param_grid, X_train, y_train, num_folds, scoring, seed )<choose_model_class>
df_all["RoofMatl"].value_counts()
kfold = KFold(n_splits=num_folds, random_state=seed )<drop_column>
df_all["RoofMatl"] = df_all["RoofMatl"].\
replace(["WdShake","WdShngl","Membran","ClyTile","Roll","Metal"],\
["Other","Other","Other","Other","Other","Other"] )
datacols = list(traindf.drop("Survived", axis=1))<drop_column>
def built_interactions(feat1, feat2):
interactions = feat1 + "_" + feat2
couples_seen = []
filtered_interactions = []
for couple in interactions.values.tolist() :
val1, val2 = couple.split("_")
 inverted_couple = val2 + "_" + val1
saw_couple = couple in couples_seen
saw_inverted_couple = inverted_couple in couples_seen
if not(saw_couple or saw_inverted_couple):
couples_seen.append(couple)
 filtered_interactions.append(couple)
 else:
if saw_couple:
filtered_interactions.append(couple)
 elif saw_inverted_couple:
filtered_interactions.append(inverted_couple)
 return filtered_interactions
exterior_couples = built_interactions(df_all["Exterior1st"], df_all["Exterior2nd"])
 df_all["Ext1_Ext2"] = exterior_couples
passengerIds = testdf["PassengerId"].copy()
testdf.drop(["PassengerId"], axis=1, inplace = True )<split>
couples_to_keep = ['VinylSd_VinylSd','MetalSd_MetalSd','HdBoard_HdBoard', 'Wd Sdng_Wd Sdng', 'Plywood_Plywood', 'CemntBd_CmentBd',
'HdBoard_Plywood', 'BrkFace_BrkFace', 'WdShing_Wd Shng', 'AsbShng_AsbShng', 'Stucco_Stucco', 'BrkFace_Wd Sdng', 'Wd Sdng_Plywood',
'Wd Sdng_Wd Shng', 'Plywood_Brk Cmn', 'Wd Sdng_HdBoard', 'WdShing_Plywood', 'VinylSd_Wd Shng', 'BrkFace_Plywood','Wd Sdng_MetalSd', 'HdBoard_ImStucc']
df_all["Ext1_Ext2"] = df_all["Ext1_Ext2"].apply(lambda couple: couple if couple in couples_to_keep else "other" )
test = testdf.values<predict_on_test>
df_all["Ext1_Ext2"].value_counts()
test_Survived_lr = pd.Series(best_lr.predict(test), name="LR")
 test_Survived_knn = pd.Series(best_knn.predict(test), name="KNN")
 test_Survived_dt = pd.Series(best_dt.predict(test), name="DT")
 test_Survived_SVC = pd.Series(best_SVC.predict(test), name="SVC")
 test_Survived_ABC = pd.Series(best_ABC.predict(test), name="ABC")
 test_Survived_RFC = pd.Series(best_RFC.predict(test), name="RFC")
 test_Survived_ETC = pd.Series(best_ETC.predict(test), name="ETC")
 test_Survived_GBC = pd.Series(best_GBC.predict(test), name="GBC")
 ensemble_results = pd.concat([test_Survived_lr, test_Survived_knn,
test_Survived_dt, test_Survived_SVC,
test_Survived_ABC, test_Survived_RFC,
test_Survived_ETC, test_Survived_GBC],axis=1)
 correlationHeatmap(ensemble_results, 'Correlation beetween models results' )<choose_model_class>
masVnrType_dict = df_all.loc[df_all["MasVnrType"]!="None",["YearBuilt","MasVnrType","MasVnrArea"]].\
groupby("YearBuilt" ).agg(pd.Series.mode)["MasVnrType"].to_dict()
mask =(df_all["MasVnrType"]=="None")&(df_all["MasVnrArea"]>0)
 df_all.loc[mask, "MasVnrType"] = df_all.loc[mask,"YearBuilt"].map(masVnrType_dict)
 df_all.loc[df_all["MasVnrArea"]==0,"MasVnrType"] = "None"
votingC = VotingClassifier(estimators=[('lr', best_lr), 
('knn', best_knn), 
('dt', best_dt), 
('svc', best_SVC), 
('abc', best_ABC), 
('rfc', best_RFC), 
('etc', best_ETC), 
('gbc', best_GBC)],
voting='hard', n_jobs=-1)
 votingC = votingC.fit(X_train, y_train )<save_to_csv>
bsmt_cols = ["BsmtQual","BsmtCond","BsmtExposure","BsmtFinType1","BsmtFinSF1",\
"BsmtFinType2","BsmtFinSF2","BsmtUnfSF","TotalBsmtSF","BsmtFullBath","BsmtHalfBath","Set","Id"]
df_all.iloc[[2217,2218]][bsmt_cols]
predictions = votingC.predict(test)
 test_Survived = pd.Series(votingC.predict(test), name="Survived")
 test_Survived = test_Survived.apply(int)
 results = pd.concat([passengerIds,test_Survived],axis=1)
 results.to_csv('submission.csv', columns=['PassengerId', 'Survived'], index=False )<load_from_csv>
df_all.loc[df_all["Id"].isin([2218,2219]),"BsmtQual"] = df_all["BsmtQual"].mode() [0]
df_train = pd.read_csv('.. /input/train.csv')
 df_test = pd.read_csv('.. /input/test.csv')
 df_combined = pd.concat([df_train, df_test], sort=False )<sort_values>
df_all["BsmtUnfSF_ratio"] = df_all["BsmtUnfSF"] / df_all["TotalBsmtSF"]
df_all["BsmtUnfSF_ratio"] = df_all["BsmtUnfSF_ratio"].fillna(0 )
df_train[['Pclass','Survived']].groupby(['Pclass'], as_index=False ).mean().sort_values(by="Survived", ascending=False )<sort_values>
df_all["Heating"].value_counts()
df_train[['Sex','Survived']].groupby(['Sex'], as_index=False ).mean().sort_values(by="Survived", ascending=False )<sort_values>
garage_cols = ["GarageType","GarageYrBlt","GarageFinish","GarageCars","GarageArea","GarageQual","GarageCond","Id"]
df_all.iloc[2126][garage_cols]
df_train[['SibSp','Survived']].groupby(['SibSp'], as_index=False ).mean().sort_values(by="Survived", ascending=False )<sort_values>
df_all.loc[df_all["Id"]==2127, "GarageFinish"] = "Unf"
df_all.loc[df_all["Id"]==2127, "GarageCond"] = "TA"
df_all.loc[df_all["Id"]==2127, "GarageQual"] = "TA"
df_all.loc[df_all["Id"]==2127, "GarageYrBlt"] = 1910
df_train[['Parch','Survived']].groupby(['Parch'], as_index=False ).maean().sort_values(by="Survived", ascending=False )<feature_engineering>
index = df_all["GarageYrBlt"].argmax()
print("Newest garage year built:",df_all["GarageYrBlt"].iloc[index])
 df_all["GarageYrBlt"].iloc[index] = df_all["YearBuilt"].iloc[index]
df_test['Title'] = df_test['Name'].str.extract('([A-Za-z]+)\.', expand=False )<categorify>
df_all["PoolQC"].value_counts()
df_train['Title'] = df_train['Title'].replace(['Capt', 'Col', 'Don', 'Dr', 'Major', 'Jonkheer', 'Sir', 'Rev', 'Dona'], 'Rare')
 df_train['Title'] = df_train['Title'].replace(['Lady', 'Countess', 'Mme'], 'Mrs')
 df_train['Title'] = df_train['Title'].replace(['Ms', 'Mlle'], 'Miss' )<categorify>
df_all.loc[df_all["PoolQC"]=="None", "PoolArea"].value_counts()
df_test['Title'] = df_test['Title'].replace(['Capt', 'Col', 'Don', 'Dr', 'Major', 'Jonkheer', 'Sir', 'Rev', 'Dona'], 'Rare')
 df_test['Title'] = df_test['Title'].replace(['Lady', 'Countess', 'Mme'], 'Mrs')
 df_test['Title'] = df_test['Title'].replace(['Ms', 'Mlle'], 'Miss' )<groupby>
cont_cols = ["LotArea","1stFlrSF", "2ndFlrSF", 'LowQualFinSF', 'GrLivArea',\
'WoodDeckSF', 'OpenPorchSF', 'EnclosedPorch', '3SsnPorch', 'ScreenPorch',\
'PoolArea', 'MiscVal','houseAge',"totalDeckPorchSF", "landSF"]
for col in cont_cols:
if col in df_all:
df_all[col] = df_all[col].apply(float )
df_train[['Title', 'Survived']].groupby(['Title'], as_index=False ).mean()<feature_engineering>
cat_cols = ["YearBuilt","YearRemodAdd",'MoSold', 'YrSold']
for col in cat_cols:
df_all[col] = df_all[col].apply(str )
df_train['FamSize'] = df_train['Parch'] + df_train['SibSp'] + 1
df_test['FamSize'] = df_test['Parch'] + df_test['SibSp'] + 1<sort_values>
ordinal_encoder = {
"LotShape": {"Reg":0, "IR1":1, "IR2":2, "IR3":3},
"Alley": {"None":0, "Grvl":1, "Pave":2},
"Utilities": {"ELO":0, "NoSeWa":1, "NoSewr":2, "AllPub":3},
"LandSlope": {"Gtl":0, "Mod":1, "Sev":2},
"ExterQual": {"Ex":5, "Gd":4, "TA":3, "Fa":2, "Po":1, "None":0},
"ExterCond": {"Ex":5, "Gd":4, "TA":3, "Fa":2, "Po":1, "None":0},
"BsmtQual": {"Ex":5, "Gd":4, "TA":3, "Fa":2, "Po":1, "None":0},
"BsmtCond": {"Ex":5, "Gd":4, "TA":3, "Fa":2, "Po":1, "None":0},
"BsmtExposure": {"Gd":3, "Av":2, "Mn":1, "No":0, "None":0},
"BsmtFinType1": {"GLQ":6,"ALQ":5,"BLQ":4,"Rec":3,"LwQ":2,"Unf":1,"None":0},
"BsmtFinType2": {"GLQ":6,"ALQ":5,"BLQ":4,"Rec":3,"LwQ":2,"Unf":1,"None":0},
"HeatingQC": {"Ex":5, "Gd":4, "TA":3, "Fa":2, "Po":1, "None":0},
"KitchenQual": {"Ex":5, "Gd":4, "TA":3, "Fa":2, "Po":1, "None":0},
"Functional": {"Typ":6,"Min1":5,"Min2":4,"Mod":3,"Maj1":2,"Maj2":1,"Sev":0},
"FireplaceQu": {"Ex":5, "Gd":4, "TA":3, "Fa":2, "Po":1, "None":0},
"GarageFinish": {"Fin":3,"RFn":2,"Unf":1,"None":0},
"GarageQual": {"Ex":5, "Gd":4, "TA":3, "Fa":2, "Po":1, "None":0},
"GarageCond": {"Ex":5, "Gd":4, "TA":3, "Fa":2, "Po":1, "None":0},
"PavedDrive": {"Y":2,"P":1,"N":0},
"PoolQC": {"Ex":5, "Gd":4, "TA":3, "Fa":2, "Po":1, "None":0},
"Fence": {"GdPrv":4,"MnPrv":3,"GdWo":2,"MnWw":1,"None":0}
}
for col in ordinal_encoder:
df_all[col] = df_all[col].map(ordinal_encoder[col] )
df_train[['FamSize','Survived']].groupby(['FamSize'], as_index=False ).mean().sort_values(by="Survived", ascending=False )<feature_engineering>
skewed_vals = df_all.select_dtypes(["float","int"] ).drop(["SalePrice","Id","SalePrice_log"],axis=1)\ 
.apply(lambda x: skew(x.dropna())).sort_values(ascending=False)
 skewed_vals
df_train['IsAlone'] = df_train["FamSize"].map(lambda s: 1 if s==1 else 0)
 df_test['IsAlone'] = df_test["FamSize"].map(lambda s: 1 if s==1 else 0 )<sort_values>
skewed_feats = skewed_vals[np.abs(skewed_vals>0.5)].index
len(skewed_feats )
df_train[['IsAlone','Survived']].groupby(['IsAlone'], as_index=False ).mean().sort_values(by="Survived", ascending=False )<sort_values>
for feat in skewed_feats:
df_all[feat] = boxcox1p(df_all[feat], 0.17 ).dropna()
df_train[['Embarked','Survived']].groupby(['Embarked'], as_index=False ).mean().sort_values(by="Survived", ascending=False )<data_type_conversions>
no_onehot = ["Set"]
df_onehot = df_all.drop(no_onehot,axis=1)
 df_all_onehot = pd.get_dummies(df_onehot,\
columns=df_onehot.select_dtypes(["object","category"] ).columns.tolist())
 print("onehot df shape: ",df_all_onehot.shape)
df_train['Fare'] = df_train['Fare'].fillna(df_train['Fare'].median())
 df_test['Fare'] = df_test['Fare'].fillna(df_test['Fare'].median() )<feature_engineering>
def rmse(pred,truth):
pred = np.squeeze(pred)
 truth = np.squeeze(truth)
 return round(np.sqrt(np.mean(np.square(pred-truth))),4 )
df_train['CategoricalFare'] = pd.qcut(df_train['Fare'], 10)
<sort_values>
def cv_rmse(model,X,y):
cv_score = cross_val_score(model, X, y, cv=5, scoring="neg_mean_squared_error")
 cv_score = np.sqrt(cv_score*(-1))
mean = cv_score.mean()
std = cv_score.std()
return cv_score, round(mean,4), round(std,4 )
print(df_train[['CategoricalFare','Survived']].groupby(['CategoricalFare'], as_index=False ).mean().sort_values(by="Survived", ascending=False))<count_missing_values>
outlier_GrLivArea = df_all_onehot.loc[(df_all_onehot["GrLivArea"]>18)&(df_all_onehot["SalePrice"]<300000),:].index.tolist()
outlier_totalHouseSF = df_all_onehot.loc[(df_all_onehot["totalHouseSF"]>20)&(df_all_onehot["SalePrice"]<200000),:].index.tolist()
outlier_1stFlrSF = df_all_onehot.loc[(df_all_onehot["1stFlrSF"]>18)&(df_all_onehot["SalePrice"]<200000),:].index.tolist()
outlier_TotalBsmtSF = df_all_onehot.loc[(df_all_onehot["TotalBsmtSF"]>18)&(df_all_onehot["SalePrice"]<200000),:].index.tolist()
age_null_count = df_train['Age'].isnull().sum()
age_null_count<feature_engineering>
outliers = outlier_1stFlrSF +\
outlier_GrLivArea +\
outlier_totalHouseSF +\
outlier_TotalBsmtSF +\
outlier_lotarea +\
outlier_lotfrontage +\
outlier_overallqual
df_train['Age'] = df_train['Age'].fillna(age_median)
 df_train['CategoricalAge'] = pd.cut(df_train['Age'], 8)
 print(df_train[['CategoricalAge', 'Survived']].groupby(['CategoricalAge'], as_index=False ).mean() )<feature_engineering>
df = df_all_onehot.loc[df_all_onehot["SalePrice"].notnull() ,:]\ 
.drop(["SalePrice", "SalePriceSF", "Id"], axis=1)
 df_removed_outliers = df.drop(outliers )
df_test['Age'] = df_test['Age'].fillna(age_median)
<categorify>
X = df_removed_outliers.drop(["SalePrice_log"], axis=1)
 y = df_removed_outliers["SalePrice_log"]
df_train['Title'] = df_train['Title'].map({"Mr": 1, "Miss": 2, "Mrs": 3, "Master": 4, "Rare": 0} ).astype(int )<categorify>
ridge = make_pipeline(RobustScaler() , Ridge(alpha=0.0005))
cv_rmse(ridge, X, y )
df_test['Title'] = df_test['Title'].map({"Mr": 1, "Miss": 2, "Mrs": 3, "Master": 4, "Rare": 0} ).astype(int )<data_type_conversions>
lasso = make_pipeline(RobustScaler() , Lasso(alpha =0.0005, random_state=1))
cv_rmse(lasso, X, y )
df_train['Sex'] = df_train['Sex'].map({'female' : 0, 'male' : 1} ).astype(int)
 df_train.head()<data_type_conversions>
elastic = make_pipeline(RobustScaler() , ElasticNet(alpha=0.0005, l1_ratio=.9, random_state=3))
cv_rmse(elastic, X, y )
df_test['Sex'] = df_test['Sex'].map({'female' : 0, 'male' : 1} ).astype(int)
 df_test.head()<data_type_conversions>
catboost = CatBoostRegressor(logging_level="Silent")
 cv_rmse(catboost, X, y )
df_train.loc[df_train['Embarked'] == 'Empty']
df_train['Embarked'] = df_train['Embarked'].fillna("Empty")
 df_train['Embarked'] = df_train['Embarked'].map({'S' : 0, 'C' : 1, 'Q' : 2, 'Empty' : 0}, na_action='ignore' ).astype(int)
 df_train.head()<data_type_conversions>
krr = make_pipeline(RobustScaler() , KernelRidge(alpha=0.6, kernel="polynomial", degree=2, coef0=2.5))
cv_rmse(krr, X, y )
df_test['Embarked'] = df_test['Embarked'].fillna("Empty")
 df_test.loc[df_test['Embarked'] == 'Empty']
df_test['Embarked'] = df_test['Embarked'].map({'S' : 0, 'C' : 1, 'Q' : 2, 'Empty' : 0}, na_action='ignore' ).astype(int)
 df_test.head()<feature_engineering>
stack = StackingRegressor(estimators=[("elastic",elastic),("KRR",krr),("Catboost",catboost)],\
final_estimator=Ridge() , cv=5)
 cv_rmse(stack, X, y )
df_train['CategoricalAge'] = 0
df_train.loc[df_train['Age'] <= 10.368, 'CategoricalAge'] = 0
df_train.loc[(df_train['Age'] > 10.368)&(df_train['Age'] <= 20.315), 'CategoricalAge'] = 1
df_train.loc[(df_train['Age'] > 20.315)&(df_train['Age'] <= 30.263), 'CategoricalAge'] = 2
df_train.loc[(df_train['Age'] > 30.263)&(df_train['Age'] <= 40.21), 'CategoricalAge'] = 3
df_train.loc[(df_train['Age'] > 40.21)&(df_train['Age'] <= 50.158), 'CategoricalAge'] = 4
df_train.loc[(df_train['Age'] > 50.158)&(df_train['Age'] <= 60.105), 'CategoricalAge'] = 5
df_train.loc[(df_train['Age'] > 60.105)&(df_train['Age'] <= 70.052), 'CategoricalAge'] = 6
df_train.loc[df_train['Age'] > 70.052, 'CategoricalAge'] = 7
df_train.head()<feature_engineering>
X_test = df_all_onehot.loc[df_all_onehot["SalePrice"].isnull() ,:]\ 
.drop(["SalePrice_log", "SalePriceSF", "Id", "SalePrice"], axis=1)
 stack.fit(X, y)
 preds = stack.predict(X_test)
 my_submission = pd.DataFrame({'Id': df_test.Id, 'SalePrice': np.expm1(preds)} )
<feature_engineering><EOS> 
my_submission.to_csv('elastic_krr_catboost_Stacked.csv', index=False )