Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<predict_on_test>
import os
import warnings
import numpy as np
import pandas as pd
import matplotlib as mpl
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.feature_selection import mutual_info_regression
from sklearn.decomposition import PCA
from sklearn.preprocessing import LabelEncoder, OneHotEncoder, OrdinalEncoder, StandardScaler
from sklearn.model_selection import train_test_split, KFold, StratifiedKFold
from sklearn.linear_model import LinearRegression, Lasso, Ridge, ElasticNet
from sklearn.ensemble import RandomForestRegressor
from lightgbm import LGBMRegressor
from catboost import CatBoostRegressor
from xgboost import XGBRegressor
from sklearn.metrics import mean_squared_error
import optuna
vote.fit(X,y)
 preds = vote.predict(test_df_final )<save_to_csv>
pd.set_option("display.max_columns", None)
 SEED = 42
TUNING_LASSO = False
TUNING_XGB = False
TRAIN_TIME = 4 * 60 * 60
N_TRIALS = 50
STUDY_NAME = 'STUDY'
submission = pd.DataFrame({'PassengerId':p_id,'Survived':preds})
 submission.to_csv('20201126_vote.csv',index=False )<import_modules>
def clean(df):
df.drop(['Id'], axis=1, inplace=True)
 miscf = ['Shed','Gar2', 'Othr']
for f in miscf:
df['MiscFeature'+f] = df.apply(lambda x: x['MiscVal'] if x['MiscFeature']==f else np.NaN , axis=1)
 df.drop(['MiscFeature','MiscVal'], axis=1, inplace=True)
 df['Exterior1st'] = df['Exterior1st'].replace({"WdShing": "Wd Shng"})
 df["Exterior2nd"] = df["Exterior2nd"].replace({"CmentBd": "CemntBd"})
 df["Exterior2nd"] = df["Exterior2nd"].replace({"Brk Cmn": "BrkComm"})
 df["BldgType"] = df["BldgType"].replace({"Twnhs": "TwnhsI"})
 df["GarageYrBlt"] = df["GarageYrBlt"].where(df.GarageYrBlt != 2207, 2007)
 return df
sns.set(style="darkgrid")
 SEED = 31
random.seed(SEED)
 for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename)) 
<load_from_csv>
def load_data() :
train = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 X_train = train.copy()
y_train = X_train['SalePrice']
test = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv')
 X_test = test.copy()
X_train = clean(X_train)
 X_test = clean(X_test)
 all_features = ['MSSubClass','MSZoning','LotFrontage','LotArea','Street','Alley','LotShape','LandContour','Utilities','LotConfig','LandSlope','Neighborhood','Condition1','Condition2','BldgType','HouseStyle','OverallQual','OverallCond','YearBuilt','YearRemodAdd','RoofStyle','RoofMatl','Exterior1st','Exterior2nd','MasVnrType','MasVnrArea','ExterQual','ExterCond','Foundation','BsmtQual','BsmtCond','BsmtExposure','BsmtFinType1','BsmtFinSF1','BsmtFinType2','BsmtFinSF2','BsmtUnfSF','TotalBsmtSF','Heating','HeatingQC','CentralAir','Electrical','1stFlrSF','2ndFlrSF','LowQualFinSF','GrLivArea','BsmtFullBath','BsmtHalfBath','FullBath','HalfBath','BedroomAbvGr','KitchenAbvGr','KitchenQual','TotRmsAbvGrd','Functional','Fireplaces','FireplaceQu','GarageType','GarageYrBlt','GarageFinish','GarageCars','GarageArea','GarageQual','GarageCond','PavedDrive','WoodDeckSF','OpenPorchSF','EnclosedPorch','3SsnPorch','ScreenPorch','PoolArea','PoolQC','Fence','MoSold','YrSold','SaleType','SaleCondition','SalePrice','MiscFeatureShed','MiscFeatureGar2','MiscFeatureOthr']
num_features = ['BsmtFullBath','BsmtHalfBath','FullBath','HalfBath','BedroomAbvGr','KitchenAbvGr','TotRmsAbvGrd','Fireplaces','GarageCars','LotFrontage','LotArea','MasVnrArea','BsmtFinSF1','BsmtFinSF2','BsmtUnfSF','TotalBsmtSF','1stFlrSF','2ndFlrSF','LowQualFinSF','GrLivArea','GarageArea','WoodDeckSF','OpenPorchSF','EnclosedPorch','3SsnPorch','ScreenPorch','PoolArea','MiscFeatureGar2','MiscFeatureOthr','MiscFeatureShed','MoSold','YearBuilt','YearRemodAdd','YrSold','GarageYrBlt']
num_continuous_features = ['LotFrontage','LotArea','MasVnrArea','BsmtFinSF1','BsmtFinSF2','BsmtUnfSF','TotalBsmtSF','1stFlrSF','2ndFlrSF','LowQualFinSF','GrLivArea', 'GarageArea','WoodDeckSF','OpenPorchSF','EnclosedPorch','3SsnPorch','ScreenPorch','PoolArea','MiscFeatureGar2','MiscFeatureOthr','MiscFeatureShed','MoSold','YearBuilt','YearRemodAdd','YrSold','GarageYrBlt']
num_discrete_features=['BsmtFullBath','BsmtHalfBath','FullBath','HalfBath','BedroomAbvGr','KitchenAbvGr','TotRmsAbvGrd','Fireplaces','GarageCars']
cat_features = ['LotShape','LandContour','OverallQual','OverallCond','Utilities','LandSlope','ExterQual','ExterCond','BsmtQual','BsmtCond','BsmtExposure','HeatingQC','KitchenQual','FireplaceQu','GarageFinish','GarageQual','GarageCond','PoolQC','Fence','MSSubClass','MSZoning','Street','Alley','LotConfig','Neighborhood','Condition1','Condition2','BldgType','HouseStyle','RoofStyle','RoofMatl','Exterior1st','Exterior2nd','MasVnrType','Foundation','BsmtFinType1','BsmtFinType2','Heating','CentralAir','Electrical','Functional','GarageType','PavedDrive','SaleType','SaleCondition']
cat_features_to_encode = cat_features.copy()
cat_features_to_encode.remove('OverallQual')
 cat_features_to_encode.remove('OverallCond')
 return train, test, X_train, y_train, X_test, all_features, num_features, num_continuous_features, num_discrete_features, cat_features, cat_features_to_encode
data = pd.read_csv("/kaggle/input/titanic/train.csv")
 data_test = pd.read_csv("/kaggle/input/titanic/test.csv" )<prepare_x_and_y>
def cv_loop(
 X_train,
y_train,
X_test,
model,
useful_features,
num_features,
cat_features,
cat_features_to_encode,
encoding = 'ohe',
new_features=[],
scaling=False,
clip=False, clipmin=np.log(34900), clipmax=np.log(755000),
tuning=True,
early_stopping=True 
):
y_train = np.log(y_train)
 num_features = np.intersect1d(num_features, useful_features)
 cat_features = np.intersect1d(cat_features, useful_features)
 cat_features_to_encode = np.intersect1d(cat_features_to_encode, useful_features)
 cum_rmse_val = 0
iteration = 1
N_SPLITS = 5
kf = KFold(n_splits=N_SPLITS, shuffle=True, random_state=42)
 for train_index, val_index in kf.split(X_train, y_train):
X_train_, X_val_ = X_train.iloc[train_index], X_train.iloc[val_index]
y_train_, y_val_ = y_train[train_index], y_train[val_index]
X_train__ = X_train_.reset_index(drop=True)
 X_val__ = X_val_.reset_index(drop=True)
 X_train__ = X_train__.loc[:, useful_features]
X_val__ = X_val__.loc[:, useful_features]
if(tuning==False):
useful_features_ = useful_features.copy()
useful_features_.remove('SalePrice')
 X_test__ = X_test.loc[:, useful_features_]
if(new_features.count('NbNAs')==1):
X_train__['NbNAs'] = X_train__.isnull().sum(axis=1)
 X_val__['NbNAs'] = X_val__.isnull().sum(axis=1)
 if(tuning==False): X_test__['NbNAs'] = X_test__.isnull().sum(axis=1)
 for col in num_features:
X_train__[col] = X_train__[col].fillna(0)
 X_val__[col] = X_val__[col].fillna(0)
 if(tuning==False): X_test__[col] = X_test__[col].fillna(0)
 for col in cat_features:
X_train__[col] = X_train__[col].fillna("None")
 X_val__[col] = X_val__[col].fillna("None")
 if(tuning==False): X_test__[col] = X_test__[col].fillna("None")
 if(new_features.count('LivLotRatio')==1):
X_train__['LivLotRatio'] = X_train__['GrLivArea'] / X_train__['LotArea']
X_val__['LivLotRatio'] = X_val__['GrLivArea'] / X_val__['LotArea']
if(tuning==False): X_test__['LivLotRatio'] = X_test__['GrLivArea'] / X_test__['LotArea']
if(new_features.count('Spaciousness')==1):
X_train__['Spaciousness'] = X_train__['GrLivArea'] / X_train__['TotRmsAbvGrd']
X_val__['Spaciousness'] = X_val__['GrLivArea'] / X_val__['TotRmsAbvGrd']
if(tuning==False): X_test__['Spaciousness'] = X_test__['GrLivArea'] / X_test__['TotRmsAbvGrd']
if(new_features.count('MedNhbdArea')==1):
feat = X_train__.groupby('Neighborhood')['GrLivArea'].median()
feat = feat.to_dict()
X_train__.loc[:,'MedNhbdArea'] = X_train__['Neighborhood'].map(feat)
 X_val__.loc[:,'MedNhbdArea'] = X_val__['Neighborhood'].map(feat)
 if(tuning==False): X_test__.loc[:,'MedNhbdArea'] = X_test__['Neighborhood'].map(feat)
 if(new_features.count('GrLivAreaInNbhd')==1):
feat = X_train__.groupby('Neighborhood')['GrLivArea'].median()
feat = feat.to_dict()
X_train__.loc[:,'GrLivAreaInNbhd'] = X_train__['Neighborhood'].map(feat)
 X_train__['GrLivAreaInNbhd'] = X_train__['GrLivArea'] - X_train__['GrLivAreaInNbhd']
X_val__.loc[:,'GrLivAreaInNbhd'] = X_val__['Neighborhood'].map(feat)
 X_val__['GrLivAreaInNbhd'] = X_val__['GrLivArea'] - X_val__['GrLivAreaInNbhd']
if(tuning==False):
X_test__.loc[:,'GrLivAreaInNbhd'] = X_test__['Neighborhood'].map(feat)
 X_test__['GrLivAreaInNbhd'] = X_test__['GrLivArea'] - X_test__['GrLivAreaInNbhd']
if(new_features.count('MedNhbdArea_Ext')==1):
feat = X_train__.groupby('Neighborhood')['LotArea'].median()
feat = feat.to_dict()
X_train__.loc[:,'MedNhbdArea_Ext'] = X_train__['Neighborhood'].map(feat)
 X_val__.loc[:,'MedNhbdArea_Ext'] = X_val__['Neighborhood'].map(feat)
 if(tuning==False): X_test__.loc[:,'MedNhbdArea_Ext'] = X_test__['Neighborhood'].map(feat)
 if(new_features.count('LotAreaInNbhd')==1):
feat = X_train__.groupby('Neighborhood')['LotArea'].median()
feat = feat.to_dict()
X_train__.loc[:,'LotAreaInNbhd'] = X_train__['Neighborhood'].map(feat)
 X_train__['LotAreaInNbhd'] = X_train__['LotArea'] - X_train__['LotAreaInNbhd']
X_val__.loc[:,'LotAreaInNbhd'] = X_val__['Neighborhood'].map(feat)
 X_val__['LotAreaInNbhd'] = X_val__['LotArea'] - X_val__['LotAreaInNbhd']
if(tuning==False):
X_test__.loc[:,'LotAreaInNbhd'] = X_test__['Neighborhood'].map(feat)
 X_test__['LotAreaInNbhd'] = X_test__['LotArea'] - X_test__['LotAreaInNbhd']
if(new_features.count('OverallQualCondProduct')==1):
X_train__['OverallQualCondProduct'] = X_train__['OverallQual'] * X_train__['OverallCond']
X_val__['OverallQualCondProduct'] = X_val__['OverallQual'] * X_val__['OverallCond']
if(tuning==False): X_test__['OverallQualCondProduct'] = X_test__['OverallQual'] * X_test__['OverallCond']
if(new_features.count('LowQualFinRatio')==1):
X_train__['LowQualFinRatio'] = X_train__['LowQualFinSF'] / X_train__['GrLivArea']
X_val__['LowQualFinRatio'] = X_val__['LowQualFinSF'] / X_val__['GrLivArea']
if(tuning==False): X_test__['LowQualFinRatio'] = X_test__['LowQualFinSF'] / X_test__['GrLivArea']
if(encoding=='ohe'):
enc = OneHotEncoder(handle_unknown = 'ignore')
 X_train__enc = pd.DataFrame(enc.fit_transform(X_train__[cat_features_to_encode] ).toarray())
 X_val__enc = pd.DataFrame(enc.transform(X_val__[cat_features_to_encode] ).toarray())
 X_train__enc.columns = enc.get_feature_names(cat_features_to_encode)
 X_val__enc.columns = enc.get_feature_names(cat_features_to_encode)
 X_train__ = X_train__.join(X_train__enc)
 X_val__ = X_val__.join(X_val__enc)
 if(tuning==False):
X_test__enc = pd.DataFrame(enc.transform(X_test__[cat_features_to_encode] ).toarray())
 X_test__enc.columns = enc.get_feature_names(cat_features_to_encode)
 X_test__ = X_test__.join(X_test__enc)
 elif(encoding=='ord'):
enc = OrdinalEncoder()
X_train__enc = pd.DataFrame(enc.fit_transform(X_train__[cat_features_to_encode]))
X_val__enc = pd.DataFrame(enc.transform(X_val__[cat_features_to_encode]))
X_train__enc.columns = [cat_features_to_encode]
X_val__enc.columns = [cat_features_to_encode]
X_train__ = X_train__.join(X_train__enc, rsuffix='_ord_enc')
 X_val__ = X_val__.join(X_val__enc, rsuffix='_ord_enc')
 if(tuning==False):
X_test__enc = pd.DataFrame(enc.transform(X_test__[cat_features_to_encode]))
X_test__enc.columns = [cat_features_to_encode]
X_test__ = X_test__.join(X_test__enc, rsuffix='_ord_enc')
 elif(encoding=='tar_enc'):
for f in cat_features_to_encode:
feat = X_train__.groupby(f)['SalePrice'].mean()
feat = feat.to_dict()
X_train__.loc[:,f"tar_enc_{f}"] = X_train__[f].map(feat)
 X_val__.loc[:,f"tar_enc_{f}"] = X_val__[f].map(feat)
 if(tuning==False): X_test__.loc[:,f"tar_enc_{f}"] = X_test__[f].map(feat)
 X_train__.drop(columns='SalePrice', inplace=True)
 X_train__.drop(columns=cat_features_to_encode, inplace=True)
 X_val__.drop(columns='SalePrice', inplace=True)
 X_val__.drop(columns=cat_features_to_encode, inplace=True)
 if(tuning==False): X_test__.drop(columns=cat_features_to_encode, inplace=True)
 X_train__.to_csv(f'X_train__{iteration}.csv', index = False)
 X_val__.to_csv(f'X_val__{iteration}.csv', index = False)
 if(tuning==False): X_test__.to_csv(f'X_test__{iteration}.csv', index = False)
 if scaling:
scaler = StandardScaler()
X_train__ = pd.DataFrame(scaler.fit_transform(X_train__), columns=X_train__.columns)
 X_val__ = pd.DataFrame(scaler.transform(X_val__), columns=X_val__.columns)
 if(tuning==False): X_test__ = pd.DataFrame(scaler.transform(X_test__), columns=X_test__.columns)
 X_train__.to_csv(f'X_train__{iteration}__scaled.csv', index = False)
 X_val__.to_csv(f'X_val__{iteration}__scaled.csv', index = False)
 if(tuning==False): X_test__.to_csv(f'X_test__{iteration}__scaled.csv', index = False)
 if early_stopping:
model.fit(X_train__, y_train_, eval_set=[(X_val__, y_val_)], early_stopping_rounds=100, verbose=False)
 else:
model.fit(X_train__, y_train_)
 y_val_preds = model.predict(X_val__)
 if(clip): y_val_preds = np.clip(y_val_preds,clipmin,clipmax)
 if(tuning==False):
if iteration==1:
oof_preds = pd.Series(data=y_val_preds,index=val_index)
 else:
oof_preds = pd.concat([oof_preds, pd.Series(data=y_val_preds,index=val_index)])
 rmse_val = mean_squared_error(y_val_, y_val_preds, squared=False)
 print(str(iteration)+ '/' + str(N_SPLITS)+ ' KFold RMSLE: ' + str(rmse_val))
cum_rmse_val = cum_rmse_val + rmse_val
if(tuning==False):
new_preds = model.predict(X_test__)
 if(clip): new_preds = np.clip(new_preds,clipmin,clipmax)
 if iteration==1:
preds = new_preds
else:
preds = preds + new_preds
iteration = iteration + 1
if(tuning==False): preds = preds/N_SPLITS
avg_rmse = cum_rmse_val/N_SPLITS
print('Average RMSLE: ' + str(avg_rmse))
if tuning:
return avg_rmse, None, None
else:
return avg_rmse, np.exp(oof_preds.sort_index()), np.exp(preds )
y = data["Survived"]
x = data.copy()
x_test = data_test.copy()
x_full = pd.concat([x, x_test])
 n = len(x)
 n_test = len(x_test)
 n_full = n + n_test<count_missing_values>
_, _, X_train, y_train, X_test, all_features, num_features, num_continuous_features, num_discrete_features, cat_features, cat_features_to_encode = load_data()
X_train.head(2 )
( x_full.drop(["Survived"], axis = 1 ).isna().sum() / n_full)* 100<filter>
feature_types = pd.DataFrame(data=[num_features, cat_features])
 feature_types.index=['Numerical','Categorical']
feature_types.style.set_table_styles([
{'selector': 'thead', 'props': [('display', 'none')]}
] )
x_full[x_full["Fare"].isnull() ]<filter>
f_with_na_train = X_train.isna().sum(axis=0)
 f_with_na_train = f_with_na_train[f_with_na_train>0]
f_with_na_train.name='Nb of NaNs in train'
f_with_na_test = X_test.isna().sum(axis=0)
 f_with_na_test = f_with_na_test[f_with_na_test>0]
f_with_na_test.name='Nb of NaNs in test'
f_with_na = pd.concat([f_with_na_train, f_with_na_test], axis=1)
 f_with_na.fillna(0, inplace=True)
 f_with_na = f_with_na[['Nb of NaNs in train','Nb of NaNs in test']]
f_with_na.sort_values(['Nb of NaNs in train', 'Nb of NaNs in test'], ascending=False)[['Nb of NaNs in train', 'Nb of NaNs in test']].plot(kind='bar', figsize=(20,4))
plt.show()
x_full[x_full["Embarked"].isnull() ]<groupby>
_, _, X_train, y_train, X_test, all_features, num_features, _, _, cat_features, cat_features_to_encode = load_data()
useful_features = [e for e in all_features if e not in('PoolArea','3SsnPorch','MoSold','YrSold','RoofMatl','Utilities','MiscFeatureGar2','PoolQC')]
new_features = ['NbNAs','LivLotRatio','Spaciousness','MedNhbdArea','GrLivAreaInNbhd','MedNhbdArea_Ext','LotAreaInNbhd','OverallQualCondProduct','LowQualFinRatio']
encoding = 'tar_enc'
scaling = True
clip = False
early_stopping = True
embarked_survived = x.groupby(["Embarked", "Survived"])["PassengerId"].count()
embarked_survived<create_dataframe>
def objective(trial):
param_grid = {
'max_depth': trial.suggest_int('max_depth', 3, 15),
'n_estimators': trial.suggest_int('n_estimators', 1000, 10000),
'eta': trial.suggest_loguniform('eta', 0.001, 0.3),
'subsample': trial.suggest_discrete_uniform('subsample', 0.1, 1.0, 0.1),
'colsample_bytree': trial.suggest_discrete_uniform('colsample_bytree', 0.2, 1.0, 0.1),
'colsample_bylevel': trial.suggest_discrete_uniform('colsample_bylevel', 0.2, 1.0, 0.1),
'min_child_weight': trial.suggest_loguniform('min_child_weight', 0.1, 10),
'reg_lambda': trial.suggest_loguniform('reg_lambda', 0.1, 100),
'reg_alpha': trial.suggest_loguniform('reg_alpha', 0.0001, 10),
'gamma': trial.suggest_loguniform('gamma', 0.001, 10),
}
model = XGBRegressor(
 tree_method='gpu_hist',
predictor='gpu_predictor',
n_jobs=4,
**param_grid 
)
 avg_rmse, _, _ = cv_loop(
 X_train=X_train,
y_train=y_train,
X_test=X_test,
model=model,
useful_features=all_features,
num_features=num_features,
cat_features=cat_features,
cat_features_to_encode=cat_features_to_encode,
encoding = encoding,
new_features=new_features,
scaling = scaling,
clip = clip, clipmin=np.log(34900), clipmax=np.log(755000),
tuning=True,
early_stopping = early_stopping 
)
 return avg_rmse
def display_survival_prob(data, feature):
feature_survived = data.groupby([feature, "Survived"])["PassengerId"].count()
classes = set(feature_survived.index.get_level_values(0))
surv_counts = {key: {} for key in classes}
for c in classes:
for survive in range(0, 2):
surv_counts[c][survive] = 0
if survive in feature_survived[c]:
surv_counts[c][survive] = feature_survived[c][survive]
df_surv = pd.DataFrame(surv_counts)
 surv_percentages = {}
for col in df_surv.columns:
surv_percentages[col] = [(count / df_surv[col].sum())* 100 for count in df_surv[col]]
df_survived_percentages = pd.DataFrame(surv_percentages ).transpose()
class_names = tuple(classes)
 bar_count = np.arange(len(class_names))
bar_width = 0.5
not_survived = df_survived_percentages[0]
survived = df_survived_percentages[1]
plt.figure(figsize=(10, 6))
plt.bar(bar_count, not_survived, color="
plt.bar(bar_count, survived, bottom=not_survived, color="
plt.xlabel("{}".format(feature), size=15, labelpad=20)
 plt.ylabel("Survival Percentage", size=15, labelpad=20)
 plt.xticks(bar_count, class_names)
 plt.tick_params(axis="x", labelsize=15)
 plt.tick_params(axis="y", labelsize=15)
 plt.legend(loc="upper left", bbox_to_anchor=(1, 1), prop={"size": 15})
 plt.title("Survival Percentage by {}".format(feature), size=18, y=1.05)
 plt.show()<groupby>
if TUNING_XGB:
study = optuna.create_study(direction='minimize', study_name=STUDY_NAME)
 study.optimize(objective, timeout=TRAIN_TIME)
 print('Number of finished trials: ', len(study.trials))
print('Best trial:')
 trial = study.best_trial
print('\tValue: {}'.format(trial.value))
print('\tParams: ')
 for key, value in trial.params.items() :
print('\t\t{}: {}'.format(key, value))
embarked_class_passengers = x.groupby("Pclass")["Embarked"].value_counts()
print(embarked_class_passengers )<prepare_output>
if TUNING_XGB:
model = XGBRegressor(
 tree_method='gpu_hist',
predictor='gpu_predictor',
n_jobs=4,
**trial.params)
 else:
params = {
'max_depth': 5,
'n_estimators': 7779,
'eta': 0.0044144556312306175,
'subsample': 0.30000000000000004,
'colsample_bytree': 0.2,
'colsample_bylevel': 0.4,
'min_child_weight': 0.21792841014662054,
'reg_lambda': 5.06808562586094,
'reg_alpha': 0.036826697275635915,
'gamma': 0.002452743312016066,
}
model = XGBRegressor(
 tree_method='gpu_hist',
predictor='gpu_predictor',
n_jobs=4,
**params)
 avg_rmse, oof_preds, preds = cv_loop(
 X_train=X_train,
y_train=y_train,
X_test=X_test,
model=model,
useful_features=all_features,
num_features=num_features,
cat_features=cat_features,
cat_features_to_encode=cat_features_to_encode,
encoding = encoding,
new_features=new_features,
scaling = scaling,
clip = clip, clipmin=np.log(34900), clipmax=np.log(755000),
tuning=False,
early_stopping = early_stopping
)
x2 = x.copy()
x2["Cabin"] = x2["Cabin"].map(lambda cabin: cabin if pd.isnull(cabin)else cabin[0])
 x2["Cabin"].sample(10 )<groupby>
submission = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/sample_submission.csv')
 submission['SalePrice'] = preds
submission.to_csv('xgb_preds.csv', index = False)
 oof_preds.to_csv('xgb_oof_preds.csv', header=False )
x2.groupby(["Cabin", "Pclass"])["PassengerId"].count()<groupby>
_, _, X_train, y_train, X_test, all_features, num_features, _, _, cat_features, cat_features_to_encode = load_data()
useful_features = [e for e in all_features if e not in('PoolArea','3SsnPorch','MoSold','YrSold','RoofMatl','Utilities','MiscFeatureGar2','PoolQC')]
new_features = ['NbNAs','LivLotRatio','Spaciousness','MedNhbdArea','GrLivAreaInNbhd','MedNhbdArea_Ext','LotAreaInNbhd','OverallQualCondProduct','LowQualFinRatio']
encoding = 'ohe'
scaling = True
clip = True
early_stopping = False
class_cabin_passengers = x2.groupby("Pclass")["Cabin"].value_counts()
print("Number of passengers without a cabin registered: {}".format(sum(x2["Cabin"].isnull())))
 print("Number of passengers without a cabin registered that survived: {}".format(sum(( x2["Survived"] == 1)&(x2["Cabin"].isnull()))))
print("Number of passengers without a cabin registered that didn't survived: {}".format(sum(( x2["Survived"] == 0)&(x2["Cabin"].isnull()))))
print("Total number of passengers that didn't survived: {}".format(sum(x["Survived"] == 0)))
 print("=================")
 print("Passengers from class 1 with cabin registered: {}".format(class_cabin_passengers[1].sum()))
print("Total number of Passengers with cabin registered: {}".format(( ~x2["Cabin"].isnull() ).sum()))
print("Number of passengers with a cabin registered that survived: {}".format(sum(( x2["Survived"] == 1)& ~(x2["Cabin"].isnull()))))
print("Total number of Passengers of class 1: {}".format(( x2["Pclass"] == 1 ).sum()))<categorify>
def objective(trial):
param_grid = {
'alpha': trial.suggest_loguniform('alpha', 0.0001, 10000),
'max_iter': trial.suggest_loguniform('max_iter', 1000, 900000),
'random_state' : 42
}
model = Lasso(
 **param_grid 
)
 avg_rmse, _, _ = cv_loop(
 X_train = X_train,
y_train = y_train,
X_test = X_test,
model = model,
useful_features = useful_features,
num_features = num_features,
cat_features = cat_features,
cat_features_to_encode = cat_features_to_encode,
encoding = encoding,
new_features = new_features,
scaling = scaling,
clip = clip, clipmin=np.log(34900), clipmax=np.log(755000),
tuning = True,
early_stopping = early_stopping 
)
 return avg_rmse
class MissingValuesTransformer(BaseEstimator, TransformerMixin):
def __init__(self):
print("In the MissingValuesTransformer init method")
 def fit(self, x, y = None):
self._median_gender_class_age = x.groupby(["Sex", "Pclass"])["Age"].median()
self._median_class_fare = x.groupby(["Pclass"])["Fare"].median()
self._count_class_embarked = x.groupby("Pclass")["Embarked"].value_counts()
return self
def transform(self, x, y = None):
x["Cabin"] = x["Cabin"].map(lambda cabin: cabin if pd.isnull(cabin)else cabin[0])
 self._count_class_cabin = x.groupby("Pclass")["Cabin"].value_counts()
x = x.apply(self._fill_na_age, axis = 1)
 x = x.apply(self._fill_na_fare, axis = 1)
 x = x.apply(self._fill_na_embarked, axis = 1)
 x = x.apply(self._fill_na_cabin, axis = 1)
 return x
def _fill_na_age(self, row):
row["Age"] = self._median_gender_class_age[row["Sex"]][row["Pclass"]] if pd.isnull(row["Age"])else row["Age"]
return row
def _fill_na_fare(self, row):
row["Fare"] = self._median_class_fare[row["Pclass"]] if pd.isnull(row["Fare"])else row["Fare"]
return row
def _fill_na_embarked(self, row):
row_class = row["Pclass"]
total_class_passengers = self._count_class_embarked[row_class].sum()
class_embarked_dist = self._count_class_embarked[row_class] / total_class_passengers
if pd.isnull(row["Embarked"]):
row["Embarked"] = random.choices(population = class_embarked_dist.index,
weights = class_embarked_dist,
k = 1)[0]
return row
def _fill_na_cabin(self, row):
row_class = row["Pclass"]
total_class_passengers = self._count_class_cabin[row_class].sum()
class_cabin_dist = self._count_class_cabin[row_class] / total_class_passengers
if pd.isnull(row["Cabin"]):
row["Cabin"] = random.choices(population = class_cabin_dist.index,
weights = class_cabin_dist,
k = 1)[0]
return row<categorify>
if TUNING_LASSO:
study = optuna.create_study(direction='minimize', study_name=STUDY_NAME)
 study.optimize(objective, n_trials=50)
 print('Number of finished trials: ', len(study.trials))
print('Best trial:')
 trial = study.best_trial
print('\tValue: {}'.format(trial.value))
print('\tParams: ')
 for key, value in trial.params.items() :
print('\t\t{}: {}'.format(key, value))
pipeline = Pipeline(steps=[ 
("preprocessor", MissingValuesTransformer())
 ])
 filled_x = pipeline.fit_transform(x)
 filled_x.sample(10 )<drop_column>
if TUNING_LASSO:
model = Lasso(**trial.params, random_state=42)
 else:
params = {
'alpha': 0.0018185000964940012,
'max_iter': 21098,
'random_state' : 42
}
model = Lasso(**params)
 avg_rmse, oof_preds, preds = cv_loop(
 X_train = X_train,
y_train = y_train,
X_test = X_test,
model = model,
useful_features = useful_features,
num_features = num_features,
cat_features = cat_features,
cat_features_to_encode = cat_features_to_encode,
encoding = encoding,
new_features = new_features,
scaling = scaling,
clip = clip, clipmin = np.log(34900), clipmax = np.log(755000),
tuning = False,
early_stopping = early_stopping
)
x2["FamilySize"] = x2["SibSp"] + x2["Parch"] + 1
x2.drop(["SibSp", "Parch"], axis = 1, inplace = True)
 x2.sample(10 )<groupby>
submission = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/sample_submission.csv')
 submission['SalePrice'] = preds
submission.to_csv('lasso_preds.csv', index = False)
 oof_preds.to_csv('lasso_oof_preds.csv', header=False )
def get_family_group(size):
if size == 1:
return "Alone"
elif size <= 4:
return "Small"
elif size <= 6:
return "Medium"
else:
return "Large"
x2["FamilySize"] = x2["FamilySize"].map(lambda s: get_family_group(s))
x2.sample(10 )<feature_engineering>
X_train, y_train = oof_preds[['xgb_oof_preds','lasso_oof_preds']], oof_preds['y_train']
xgb_preds = pd.read_csv('xgb_preds.csv' ).iloc[:,1]
lasso_preds = pd.read_csv('lasso_preds.csv' ).iloc[:,1]
X_test = pd.concat([np.log(xgb_preds), np.log(lasso_preds)], axis=1 )
x2["NameTitle"] = x2["Name"].map(lambda name: name.split(", ")[1].split(".")[0])
 x2.sample(5 )<categorify>
metamodel = LinearRegression()
cum_rmse_val = 0
iteration = 1
kf = KFold(n_splits=5, shuffle=True, random_state=42)
 for train_index, val_index in kf.split(X_train, y_train):
X_train_, X_val_ = X_train.iloc[train_index], X_train.iloc[val_index]
y_train_, y_val_ = y_train[train_index], y_train[val_index]
metamodel.fit(X_train_, y_train_)
 y_val_preds = metamodel.predict(X_val_)
 rmse_val = mean_squared_error(y_val_, y_val_preds, squared=False)
 print(rmse_val)
 cum_rmse_val = cum_rmse_val + rmse_val
new_preds=metamodel.predict(X_test)
 if(iteration==1):
preds = new_preds
else:
preds = preds + new_preds
iteration = iteration+1
print(cum_rmse_val/5)
 preds = preds / 5
<define_search_space><EOS> 
submission = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/sample_submission.csv')
 submission['SalePrice'] = np.exp(preds)
 submission.to_csv('stacking_preds.csv', index = False )