Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<feature_engineering>
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
all_data['HaveCabin'] = 0;
all_data.loc[all_data['Cabin'].notnull() , 'HaveCabin'] = 1<groupby>
sns.set_style('darkgrid')
 pd.set_option('display.max_columns', 500 )
all_data['AnyoneSurvivedFromGroup'] = 0.5
for _, grp_df in all_data.groupby('Ticket'):
if(len(grp_df)!= 1):
for ind, row in grp_df.iterrows() :
if(row['AnyoneSurvivedFromGroup'] == 0)|(row['AnyoneSurvivedFromGroup']== 0.5):
smax = grp_df.drop(ind)['Survived'].max()
smin = grp_df.drop(ind)['Survived'].min()
passID = row['PassengerId']
if(smax == 1.0):
all_data.loc[all_data['PassengerId'] == passID, 'AnyoneSurvivedFromGroup'] = 1
elif(smin == 0.0):
all_data.loc[all_data['PassengerId'] == passID, 'AnyoneSurvivedFromGroup'] = 0<count_values>
train = pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/train.csv")
 test = pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/test.csv" )
all_data[all_data['AnyoneSurvivedFromGroup'] != 0.5]['PassengerId'].count()<groupby>
tmp = pd.isnull(train ).sum()
tmp = tmp.sort_values(ascending=False)
 tmp = tmp[tmp>0]
plt.xticks(rotation=90)
 sns.barplot(x=tmp.index, y=tmp)
 plt.show()
for _, grp_df in all_data.groupby('Cabin'):
if(len(grp_df)!= 1):
for ind, row in grp_df.iterrows() :
if(row['AnyoneSurvivedFromGroup'] == 0)|(row['AnyoneSurvivedFromGroup']== 0.5):
smax = grp_df.drop(ind)['Survived'].max()
smin = grp_df.drop(ind)['Survived'].min()
passID = row['PassengerId']
if(smax == 1.0):
all_data.loc[all_data['PassengerId'] == passID, 'AnyoneSurvivedFromGroup'] = 1
elif(smin == 0.0):
all_data.loc[all_data['PassengerId'] == passID, 'AnyoneSurvivedFromGroup'] = 0<count_values>
num_col = []
cat_col = []
for column in train.columns[~pd.Series(train.columns ).isin(['Id', 'SalePrice'])]:
if type(train[column][0])is not str:
num_col.append(column)
 else:
cat_col.append(column)
 tmp = ['MSSubClass', 'Alley', 'FireplaceQu', 'PoolQC', 'Fence', 'MiscFeature', 'MoSold']
num_col = list(set(num_col)- set(tmp))
cat_col += tmp
for col in tmp:
train[col] = train[col].astype(str )
all_data[all_data['AnyoneSurvivedFromGroup'] != 0.5]['PassengerId'].count()<count_missing_values>
col_corr = []
for col in num_col:
na_idx = pd.isnull(train[col])
 corr = np.corrcoef(x= train[col][~na_idx], y=train['SalePrice'][~na_idx])[0,1]
col_corr.append(( col, corr))
col_corr.sort(key=lambda x : -abs(x[1]))
all_data.Fare.isnull().sum()<groupby>
col_aov = []
for col in cat_col:
result = ols('SalePrice ~ {}'.format(col), data = train ).fit()
aov_table = sm.stats.anova_lm(result)
 aov_pr = aov_table['PR(>F)'][0]
col_aov.append(( col, aov_pr))
col_aov.sort(key = lambda x: x[1] )
pclass_grouped = all_data.groupby(['Pclass'])
 for pclass in all_data.Pclass.unique() :
all_data.loc[(all_data.Fare.isnull())&(all_data.Pclass == pclass), 'Fare'] = pclass_grouped.get_group(pclass ).Fare.mean()<categorify>
high_cor_list = []
tmp = corr_table[abs(corr_table)>0.4]
for col in tmp.columns:
for row in tmp[col][~pd.isnull(tmp[col])].index:
if col == row:
break
high_cor_list.append(( col,row, tmp[col][row]))
high_cor_list.sort(key = lambda x : -x[2])
 high_cor_list
all_data['FareBin'] = pd.qcut(all_data['Fare'], 4)
 label = preprocessing.LabelEncoder()
all_data['FareBin'] = label.fit_transform(all_data['FareBin'] )<count_missing_values>
high_cor_list = []
tmp = corr_table[abs(corr_table)>0.4]
for col in tmp.columns:
for row in tmp[col][~pd.isnull(tmp[col])].index:
if col == row:
break
high_cor_list.append(( col,row, tmp[col][row]))
high_cor_list.sort(key = lambda x : -x[2])
 high_cor_list
all_data.isnull().sum()<feature_engineering>
scaler = StandardScaler()
tmp = scaler.fit_transform(train[['SalePrice']])
 tmp_sorted = sorted(np.squeeze(tmp))
for a, b in zip(tmp_sorted[:10], tmp_sorted[-10:]):
print('{} {} {}'.format(round(a, 5), ' '*10, round(b,5)) )
all_data.loc[all_data['Embarked'].isnull() , 'Embarked'] = all_data['Embarked'].mode() [0]<concatenate>
train[tmp>7]
one_hot_encoding_list.append('Embarked' )<drop_column>
train = pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/train.csv")
 test = pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/test.csv" )
all_data.drop(['PassengerId', 'Name', 'Ticket', 'Cabin', 'Age', 'Fare', 'SibSp', 'Parch', 'FamilySize', 'FareBin'], axis = 1, inplace = True )<categorify>
test['SalePrice'] = 0
data = pd.concat([train,test], axis=0)
 data.reset_index(drop=True, inplace=True )
sex_mappings = {'male': 0,'female': 1}
all_data['Sex'].replace(sex_mappings, inplace = True )<concatenate>
data['MSZoning'].fillna(data['MSZoning'].mode() [0], inplace=True)
 data['Utilities'].fillna(data['Utilities'].mode() [0], inplace=True)
 data['Exterior1st'].fillna(data['Exterior1st'].mode() [0], inplace=True)
 data['Exterior2nd'].fillna(data['Exterior2nd'].mode() [0], inplace=True)
 data['MasVnrType'].fillna('None', inplace=True)
 data['MasVnrArea'].fillna(0, inplace=True)
 data['BsmtQual'].fillna('None', inplace=True)
 data['BsmtCond'].fillna('None', inplace=True)
 data['BsmtExposure'].fillna('None', inplace=True)
 data['BsmtFinType1'].fillna('None', inplace=True)
 data['BsmtFinSF1'].fillna(0, inplace=True)
 data['BsmtFinType2'].fillna('None', inplace=True)
 data['BsmtFinSF2'].fillna(0, inplace=True)
 data['BsmtUnfSF'].fillna(0, inplace=True)
 data['TotalBsmtSF'].fillna(0, inplace=True)
 data['Electrical'].fillna(data['Electrical'].mode() [0], inplace=True)
 data['BsmtFullBath'].fillna(0, inplace=True)
 data['BsmtHalfBath'].fillna(0, inplace=True)
 data['KitchenQual'].fillna(data['KitchenQual'].mode() [0], inplace=True)
 data['Functional'].fillna('None', inplace=True)
 data['GarageType'].fillna('None', inplace=True)
 data['GarageFinish'].fillna('None', inplace=True)
 data['GarageYrBlt'].fillna(data['GarageYrBlt'].mode() [0], inplace=True)
 data['GarageArea'].fillna(0, inplace=True)
 data['GarageCars'].fillna(0, inplace=True)
 data['GarageQual'].fillna('None', inplace=True)
 data['SaleType'].fillna(data['SaleType'].mode() [0], inplace=True)
 data['GarageCond'].fillna('None', inplace=True)
 data['PoolQC'].fillna('None', inplace=True)
 data['Fence'].fillna('None', inplace=True)
 data['MiscFeature'].fillna('None', inplace=True)
 data['LotFrontage'].fillna(0, inplace=True)
 data['Alley'].fillna('None', inplace=True)
 data['FireplaceQu'].fillna('None', inplace=True )
one_hot_encoding_list.append('Pclass' )<categorify>
cols = ['MiscFeature', 'Alley', 'PoolQC', 'Fence', 'FireplaceQu', 'GarageCond', 'LotFrontage']
data.drop(cols, axis=1, inplace=True )
all_data = pd.get_dummies(data = all_data, columns = one_hot_encoding_list )<split>
data['Total_SF'] = data['TotalBsmtSF'] + data['1stFlrSF'] + data['2ndFlrSF']
data['Total_Bath'] = data['BsmtFullBath'] + data['BsmtHalfBath']*0.5 + data['FullBath'] + data['HalfBath']*0.5
data['Total_Footage'] = data['BsmtFinSF1'] + data['BsmtFinSF2'] + data['1stFlrSF'] + data['2ndFlrSF']
data['Age'] = 2020 - data['YearBuilt']
data['Age_rmd'] = 2020 - data['YearRemodAdd']
data['Age_Garage'] = 3020 - data['GarageYrBlt']
data['qul_grliv'] = data['OverallQual'] * data['GrLivArea']
data['garage'] = data['GarageCars'] * data['GarageArea']
data['Years'] = data['YearBuilt'] + data['YearRemodAdd'] + data['GarageYrBlt']
data['Age_not_rmd'] = data['Age'] - data['Age_rmd']
data['Overall'] = data['OverallQual'] + data['OverallCond']
data['porch'] = data['OpenPorchSF'] + data['EnclosedPorch'] + data['ScreenPorch']
data['haspool'] = data['PoolArea'].apply(lambda x: 1 if x > 0 else 0)
 data['has2ndfloor'] = data['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)
 data['hasgarage'] = data['GarageArea'].apply(lambda x: 1 if x > 0 else 0)
 data['hasbsmt'] = data['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 data['hasfireplace'] = data['Fireplaces'].apply(lambda x: 1 if x > 0 else 0)
 data['Season'] = data['MoSold'].apply(lambda x: 'Spring' if x<= 3 else('Summer' if x <=6 else('Fall' if x <=9 else 'Winter')))
 data.drop(['MoSold'], axis=1, inplace=True )
train_data = all_data[:len(train)]
test_data = all_data[len(train):]<prepare_x_and_y>
to_cat = ['MSSubClass']
data[to_cat] = data[to_cat].astype(str )
X = train_data.iloc[: , 1:].values
y = train_data.iloc[:, 0].values
test_data = test_data.iloc[: , 1:]
X_test = test_data.values
print(str(X.shape))
print(str(X_test.shape))<choose_model_class>
num_col = []
cat_col = []
for col in data.columns:
if col == "Id" or col == 'SalePrice':
continue
elif data[col].dtype == 'object':
cat_col.append(col)
 else:
num_col.append(col)
 print(len(num_col))
print(len(cat_col))
print(len(data.columns))
def create_model() :
model = Sequential()
model.add(Dense(16, input_dim = X.shape[1], activation = 'relu', kernel_regularizer=l2(0.01)))
 model.add(Dense(16, activation = 'relu', kernel_regularizer=l2(0.01)))
 model.add(Dense(1, activation = 'sigmoid'))
model.compile(loss='binary_crossentropy', optimizer = 'adam', metrics = ['accuracy'])
 return model<train_model>
for col in cat_col:
data = pd.concat([data,pd.get_dummies(data[col], prefix=(col)) ], axis=1)
 data.drop([col], axis=1, inplace=True )
num_epochs = max_val_acc_epoch
kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=seed)
 cvscores = []
for train, test in kfold.split(X, y):
model = create_model()
history = model.fit(X[train], y[train], epochs=num_epochs, batch_size=10, verbose = 0)
 scores = model.evaluate(X[test], y[test], verbose=0)
 print("%s: %.2f%%" %(model.metrics_names[1], scores[1]*100))
cvscores.append(scores[1] * 100)
 print("%.2f%%(+/- %.2f%%)" %(np.mean(cvscores), np.std(cvscores)) )<train_model>
skews = []
for col in num_col:
skews.append(( col, skew(data[col])))
 skews.sort(key=lambda x : -abs(x[1]))
for col, value in skews:
if abs(value)>0.5:
data[col] = boxcox1p(data[col], 0.15)
 data['SalePrice'] = np.log1p(data['SalePrice'] )
model = create_model()
history = model.fit(X, y, epochs=num_epochs, batch_size=10, verbose = 0 )<predict_on_test>
cols = ['GarageArea', 'GarageYrBlt', 'TotRmsAbvGrd', '2ndFlrSF', 'BsmtFullBath']
data.drop(cols, axis=1, inplace=True)
 num_col = list(set(num_col)-set(cols))
prediction = model.predict(X_test )<create_dataframe>
cols = []
for col in data.columns:
major_ratio = data[col].value_counts().iloc[0]/len(data[col])
 if major_ratio > 0.999:
cols.append(col)
 data.drop(cols, axis=1, inplace=True)
 print('
submission = pd.DataFrame(test_copy[['PassengerId']])
 submission['Survived'] = prediction
submission['Survived'] = submission['Survived'].apply(lambda x: 0 if x < 0.5 else 1 )<save_to_csv>
data.reset_index(drop=True, inplace=True)
 train = data[data['Id']<=1460]
test = data[data['Id'] >1460]
X = list(set(train.columns)- set(['SalePrice']))
y = 'SalePrice'
submission.to_csv('submission.csv', index = False )<import_modules>
def rmse_cv(model):
result = cross_val_score(cv=5, estimator=model, X=train[X].values, y=train[y].values, scoring='neg_mean_squared_error')
 result =(-result)**(1/2)
 return np.mean(result), np.std(result)
print(os.listdir(".. /input")) 
<load_from_csv>
def rmsle(pred, actual):
result = 0
for i in range(len(pred)) :
result +=(pred[i] - actual[i])**2
result /= len(pred)
 return result**(1/2 )
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )<load_from_csv>
EN = make_pipeline(RobustScaler() , ElasticNet(alpha=1e-3, l1_ratio=0.7, max_iter=1e+4))
rmse_cv(EN)
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )<drop_column>
lasso = make_pipeline(RobustScaler() , Lasso(alpha=5e-04, max_iter=1e+04))
rmse_cv(lasso )
train.drop(['Id'], axis=1, inplace=True)
 test.drop(['Id'], axis=1, inplace=True )<prepare_x_and_y>
KRR = KernelRidge(alpha=1e-2, kernel='polynomial', degree=1, coef0=2)
 rmse_cv(KRR )
train = train[train.GrLivArea < 4500]
train.reset_index(drop=True, inplace=True)
 train["SalePrice"] = np.log1p(train["SalePrice"])
 y = train['SalePrice'].reset_index(drop=True )<concatenate>
GBoost = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05,
max_depth=4, max_features='sqrt',
min_samples_leaf=30, min_samples_split=10,
loss='huber', random_state =5)
train_features = train.drop(['SalePrice'], axis=1)
 test_features = test
features = pd.concat([train_features, test_features] ).reset_index(drop=True )<data_type_conversions>
model_xgb = xgb.XGBRegressor(colsample_bytree=0.3, gamma=0.0468,
learning_rate=0.05, max_depth=3,
min_child_weight=1.3, n_estimators=1024,
reg_alpha=0.3, reg_lambda=0.4,
subsample=0.5, silent=1,
random_state =7, nthread = -1)
features['MSSubClass'] = features['MSSubClass'].apply(str)
 features['YrSold'] = features['YrSold'].astype(str)
 features['MoSold'] = features['MoSold'].astype(str)
 features['Functional'] = features['Functional'].fillna('Typ')
 features['Electrical'] = features['Electrical'].fillna("SBrkr")
 features['KitchenQual'] = features['KitchenQual'].fillna("TA")
 features["PoolQC"] = features["PoolQC"].fillna("None")
 features['Exterior1st'] = features['Exterior1st'].fillna(features['Exterior1st'].mode() [0])
 features['Exterior2nd'] = features['Exterior2nd'].fillna(features['Exterior2nd'].mode() [0])
 features['SaleType'] = features['SaleType'].fillna(features['SaleType'].mode() [0])
<categorify>
model_lgb = lgb.LGBMRegressor(objective='regression',num_leaves=4,
learning_rate=0.05, n_estimators=1024,
max_bin = 40, bagging_fraction = 0.9,
bagging_freq = 20, feature_fraction = 0.1,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf = 2, min_sum_hessian_in_leaf = 1, random_state=5)
for col in('GarageYrBlt', 'GarageArea', 'GarageCars'):
features[col] = features[col].fillna(0)
 for col in ['GarageType', 'GarageFinish', 'GarageQual', 'GarageCond']:
features[col] = features[col].fillna('None')
 for col in('BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2'):
features[col] = features[col].fillna('None')
<groupby>
def baseline_model() :
model = Sequential()
model.add(Dense(64, input_dim=len(X), kernel_initializer='normal', activation='relu'))
model.add(Dense(8, input_dim=64, kernel_initializer='normal', activation='relu'))
model.add(Dense(1, kernel_initializer='normal'))
opt = optimizers.Adam(learning_rate = 0.005)
 model.compile(loss='mean_squared_error', optimizer=opt)
 return model
nn = KerasRegressor(build_fn=baseline_model, epochs=3000, batch_size=len(train), verbose=0)
features['MSZoning'] = features.groupby('MSSubClass')['MSZoning'].transform(lambda x: x.fillna(x.mode() [0]))<categorify>
rf = RandomForestRegressor(n_estimators=128, min_samples_split=4, min_samples_leaf = 2,
random_state=42)
 rmse_cv(rf )
objects = []
for i in features.columns:
if features[i].dtype == object:
objects.append(i)
 features.update(features[objects].fillna('None'))
print(objects)
<data_type_conversions>
class Meta_Regressor(BaseEstimator):
def __init__(self, base_models, meta_models):
self.base_models = base_models
self.meta_models = meta_models
def fit(self, X, y):
self.base_models_ = [[] for _ in self.base_models]
self.meta_models_ = clone(self.meta_models)
 Kf = KFold(n_splits=5, shuffle=True, random_state=5)
 out_fold_pred = np.zeros(( len(X), len(self.base_models)))
 for i, model in enumerate(self.base_models):
for train_idx, val_idx in Kf.split(X):
model = clone(self.base_models[i])
 model.fit(X[train_idx], y[train_idx])
 pred = model.predict(X[val_idx])
 out_fold_pred[val_idx, i] = pred
self.base_models_[i].append(model)
 self.meta_models_.fit(X=out_fold_pred, y=y)
 def predict(self, X):
meta_features = np.column_stack([
np.column_stack([model.predict(X)for model in sub_models] ).mean(axis=1)
 for sub_models in self.base_models_])
 scores = self.meta_models_.predict(meta_features)
 return scores
features['LotFrontage'] = features.groupby('Neighborhood')['LotFrontage'].transform(lambda x: x.fillna(x.median()))
numeric_dtypes = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
numerics = []
for i in features.columns:
if features[i].dtype in numeric_dtypes:
numerics.append(i)
 features.update(features[numerics].fillna(0))
numerics[1:10]<feature_engineering>
class Weighted_Ensemble(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, models):
self.models = models
def fit(self, X, y):
self.models_ = [clone(model)for model in self.models]
for model in self.models_:
model.fit(X, y)
 def predict(self, x):
results = np.zeros(len(x))
scores = [model.predict(x)for model in self.models_]
return scores
numeric_dtypes = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
numerics2 = []
for i in features.columns:
if features[i].dtype in numeric_dtypes:
numerics2.append(i)
 skew_features = features[numerics2].apply(lambda x: skew(x)).sort_values(ascending=False)
 high_skew = skew_features[skew_features > 0.5]
skew_index = high_skew.index
for i in skew_index:
features[i] = boxcox1p(features[i], boxcox_normmax(features[i] + 1))<feature_engineering>
meta_regressor = Meta_Regressor(base_models= [lasso, KRR, GBoost, model_xgb, model_lgb, nn, rf], meta_models = EN)
 weighted_ensemble = Weighted_Ensemble(models= [lasso, KRR, GBoost, model_xgb, model_lgb, nn, rf, EN, meta_regressor])
 train_x, test_x, train_y, test_y = train_test_split(train[X].values, train[y].values, test_size=0.3, random_state=4)
 weighted_ensemble.fit(train_x, train_y)
 scores = weighted_ensemble.predict(test_x )
features = features.drop(['Utilities', 'Street', 'PoolQC',], axis=1)
 features['YrBltAndRemod']=features['YearBuilt']+features['YearRemodAdd']
features['TotalSF']=features['TotalBsmtSF'] + features['1stFlrSF'] + features['2ndFlrSF']
features['Total_sqr_footage'] =(features['BsmtFinSF1'] + features['BsmtFinSF2'] +
features['1stFlrSF'] + features['2ndFlrSF'])
 features['Total_Bathrooms'] =(features['FullBath'] +(0.5 * features['HalfBath'])+
features['BsmtFullBath'] +(0.5 * features['BsmtHalfBath']))
features['Total_porch_sf'] =(features['OpenPorchSF'] + features['3SsnPorch'] +
features['EnclosedPorch'] + features['ScreenPorch'] +
features['WoodDeckSF'])
<feature_engineering>
weights = [0.1, 0.2, 0.2, 0.06, 0.09, 0.01, 0.04, 0.2, 0.1]
score = np.zeros(len(scores[0]))
for i, model in enumerate(scores):
score += scores[i] * weights[i]
rmsle(score, test_y )
features['haspool'] = features['PoolArea'].apply(lambda x: 1 if x > 0 else 0)
 features['has2ndfloor'] = features['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)
 features['hasgarage'] = features['GarageArea'].apply(lambda x: 1 if x > 0 else 0)
 features['hasbsmt'] = features['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 features['hasfireplace'] = features['Fireplaces'].apply(lambda x: 1 if x > 0 else 0 )<categorify>
meta_regressor = Meta_Regressor(base_models= [lasso, KRR, GBoost, model_xgb, model_lgb, nn, rf], meta_models = EN)
 weighted_ensemble = Weighted_Ensemble(models= [KRR, GBoost, model_xgb, EN])
 weighted_ensemble.fit(train[X].values, train[y].values)
 weights = [0.2, 0.2, 0.05,0.55]
scores = weighted_ensemble.predict(test[X].values)
 score = np.zeros(len(scores[0]))
for i, model in enumerate(scores):
score += scores[i] * weights[i]
pred = np.expm1(score )
<train_model><EOS> 
submission = pd.DataFrame({
'Id': range(1461, 2920),
'SalePrice': pred
})
 submission.to_csv("submission.csv", index=False )