Kaggle competition: House Prices - Advanced Regression Techniques
avg_preds =(xgb_preds + cat_preds + lgb_preds)/ 3.0
avg_sample_subm = pd.read_csv('.. /input/sample_submission.csv')
 avg_sample_subm['Target'] = np.argmax(avg_preds, axis=1 ).astype(int)
 avg_sample_subm['Target'] = avg_sample_subm['Target'].apply(lambda x: x+1)
 avg_sample_subm.to_csv('avg_preds.csv', index=False)
 sns.countplot(avg_sample_subm['Target'])
 plt.show()<feature_engineering>
%matplotlib inline
pd.pandas.set_option('display.max_columns',None)
<import_modules>
from sklearn.linear_model import ElasticNetCV, LassoCV, RidgeCV
from sklearn.ensemble import GradientBoostingRegressor
from sklearn.svm import SVR
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import RobustScaler
from sklearn.model_selection import KFold, cross_val_score
from sklearn.metrics import mean_squared_error, mean_squared_log_error, mean_absolute_error
from mlxtend.regressor import StackingCVRegressor
from xgboost import XGBRegressor
import numpy as np
import pandas as pd
import lightgbm as lgb
from sklearn.model_selection import StratifiedKFold<load_from_csv>
elasticnet_alphas = [5e-5, 1e-4, 5e-4, 1e-3]
elasticnet_l1ratios = [0.8, 0.85, 0.9, 0.95, 1]
lasso_alphas = [5e-5, 1e-4, 5e-4, 1e-3]
ridge_alphas = [13.5, 14, 14.5, 15, 15.5]
MODELS = {
"elasticnet" : make_pipeline(RobustScaler() , ElasticNetCV(max_iter=1e7, alphas=elasticnet_alphas, l1_ratio=elasticnet_l1ratios)) ,
"lasso" : make_pipeline(RobustScaler() , LassoCV(max_iter=1e7, alphas=lasso_alphas, random_state=42)) ,
"ridge" : make_pipeline(RobustScaler() , RidgeCV(alphas=ridge_alphas)) ,
"gradb" : GradientBoostingRegressor(n_estimators=6000, learning_rate=0.01,
max_depth=4, max_features='sqrt',
min_samples_leaf=15, min_samples_split=10,
loss='huber', random_state=42),
"svr" : make_pipeline(RobustScaler() ,
SVR(C=20, epsilon=0.008, gamma=0.0003)) ,
"xgboost" : XGBRegressor(learning_rate=0.01, n_estimators=6000,
max_depth=3, min_child_weight=0,
gamma=0, subsample=0.7,
colsample_bytree=0.7,
objective='reg:squarederror', nthread=-1,
scale_pos_weight=1, seed=27,
reg_alpha=0.00006, random_state=42)}
MODELS_stack = StackingCVRegressor(regressors=(MODELS['elasticnet'], MODELS['gradb'], MODELS['lasso'],
MODELS['ridge'], MODELS['svr'], MODELS['xgboost']),
meta_regressor=MODELS['xgboost'],
use_features_in_secondary=True )
df = pd.read_csv(".. /input/train.csv" )<categorify>
def feature_engineering(df):
temporal_features = [feature for feature in df.columns if 'Yr' in feature or 'Year' in feature or 'Mo' in feature]
numeric_features = [feature for feature in df.columns if df[feature].dtype != 'O' and feature not in temporal_features and feature not in("Id", "kfold","SalePrice")]
categorical_features = [feature for feature in df.columns if df[feature].dtype == 'O' and feature not in temporal_features]
for feature in temporal_features:
if feature == 'YrSold' or feature == 'MoSold':
pass
else:
df[feature] = df['YrSold'] - df[feature]
df['YrSold'] = df['YrSold'].astype(str)
 df['MoSold'] = df['MoSold'].astype(str)
 df['MSSubClass'] = df['MSSubClass'].apply(str)
 for feature in numeric_features:
df[feature] = df.groupby("Neighborhood")[feature].transform(lambda x: x.fillna(x.median()))
for feature in categorical_features:
df[feature] = df[feature].fillna("Missing")
 for feature in temporal_features:
if feature == 'YrSold' or feature == 'MoSold':
df[feature] = df[feature].fillna("Missing")
 else:
df[feature] = df[feature].fillna(0)
 df['TotalHouseSF'] = df['TotalBsmtSF'] + df['1stFlrSF'] + df['2ndFlrSF']
df['TotalLot'] = df['LotFrontage'] + df['LotArea']
df['TotalBsmtFin'] = df['BsmtFinSF1'] + df['BsmtFinSF2']
df['TotalBath'] = df['FullBath'] + df['HalfBath']
df['TotalPorch'] = df['OpenPorchSF'] + df['EnclosedPorch'] + df['ScreenPorch']
cols =('FireplaceQu', 'BsmtQual', 'BsmtCond', 'GarageQual', 'GarageCond',
'ExterQual', 'ExterCond','HeatingQC', 'PoolQC', 'KitchenQual', 'BsmtFinType1',
'BsmtFinType2', 'Functional', 'Fence', 'BsmtExposure', 'GarageFinish', 'LandSlope',
'LotShape', 'PavedDrive', 'Street', 'Alley', 'CentralAir', 'MSSubClass', 'YrSold', 'MoSold')
 for c in cols:
lbl = LabelEncoder()
lbl.fit(list(df[c].values))
df[c] = lbl.transform(list(df[c].values))
df["TotalGarageQual"] = df["GarageQual"] * df["GarageCond"]
df["TotalExteriorQual"] = df["ExterQual"] * df["ExterCond"]
numeric_feats = [feature for feature in df.columns if df[feature].dtype != "object" and feature not in("Id", "kfold","SalePrice")]
skewed_feats = df[numeric_feats].apply(lambda x: skew(x)).sort_values(ascending=False)
 skewness = pd.DataFrame({'Skew' :skewed_feats})
 skewness = skewness[abs(skewness)> 0.75]
skewed_features = skewness.index
lam = 0.15
for feat in skewed_features:
df[feat] = boxcox1p(df[feat], lam)
 features = [feature for feature in df.columns if df[feature].dtype == 'O']
for feature in features:
abc = df[feature].value_counts().to_dict()
for key, value in abc.items() :
if value/len(df[feature])< 0.01:
df.loc[:,feature][df[feature]==key]="rare"
return df
def preprocess(df):

 def mk_categoricals(df, prefixes=None, subsets=None):

 def mk_category(dummies):
assert(dummies.sum(axis=1)<= 1 ).all()
nans = dummies.sum(axis=1)!= 1
if nans.any() :
dummies = dummies.assign(_na=nans.astype(int))
return dummies.idxmax(axis=1 ).astype('category')
 categoricals = pd.DataFrame()
if prefixes:
for prefix in prefixes:
columns = df.columns[df.columns.str.startswith(prefix)]
categoricals[prefix] = mk_category(df[columns])
 if subsets:
for feature_name, subset in subsets.items() :
categoricals[feature_name] = mk_category(df[subset])
 return categoricals
groupper = df.groupby('idhogar')
 interactions =(pd.DataFrame(dict(
 head_escolari=df.parentesco1 * df.escolari,
head_female=df.parentesco1 * df.female,
head_partner_escolari=df.parentesco2 * df.escolari)) 
.groupby(df.idhogar)
.max())
 my_features =(groupper.mean() [['escolari', 'age', 'hogar_nin',
'hogar_total', 'epared3', 'epared1',
'etecho3', 'etecho1', 'eviv3', 'eviv1',
'male',
'r4h1', 'r4h2', 'r4h3', 'r4m1', 'r4m2',
'r4m3',
'r4t1', 'r4t2', 'r4t3', 'v2a1', 'rooms',
'bedrooms',
'meaneduc',
'SQBdependency', 'rez_esc', 'refrig',
'tamviv', 'overcrowding']] 
.join(groupper.std() [['escolari', 'age']],
rsuffix='_std')
.join(groupper[['escolari', 'age']].min() , rsuffix="_min")
.join(groupper[['escolari', 'age']].max() , rsuffix="_max")
.join(groupper[['dis']].sum() , rsuffix="_sum")
.assign(child_rate=lambda x: x.hogar_nin / x.hogar_total,
wrf=lambda x: x.epared3 - x.epared1 +
x.etecho3 - x.etecho1 +
x.eviv3 - x.eviv1,
escolari_range=lambda x: x.escolari_max - x.escolari_min,
age_range=lambda x: x.age_max - x.age_min,
rent_per_individual=lambda x: x.v2a1 / x.r4t3,
rent_per_child=lambda x: x.v2a1 / x.r4t1,
rent_per_over65=lambda x: x.v2a1 / x.r4t3,
rent_per_room=lambda x: x.v2a1 / x.rooms,
rent_per_bedroom=lambda x: x.v2a1 / x.bedrooms,
rooms_per_individual=lambda x: x.rooms / x.r4t3,
rooms_per_child=lambda x: x.rooms / x.r4t1,
bedrooms_per_individual=lambda x: x.bedrooms / x.r4t3,
bedrooms_per_child=lambda x: x.bedrooms / x.r4t1,
years_schooling_per_individual=lambda x: x.escolari / x.r4t3,
years_schooling_per_adult=lambda x: x.escolari /(x.r4t3 - x.r4t1),
years_schooling_per_child=lambda x: x.escolari / x.r4t3 
)
.drop(['hogar_nin', 'hogar_total', 'epared3', 'epared1',
'etecho3', 'etecho1', 'eviv3', 'eviv1'],
axis=1)
.join(interactions)
.join(groupper[['computer', 'television',
'qmobilephone', 'v18q1']] 
.mean().sum(axis=1 ).rename('technics')) 
.assign(technics_per_individual=lambda x: x.technics / x.r4t3,
technics_per_child=lambda x: x.technics / x.r4t1)
.join(mk_categoricals(groupper.mean() ,
prefixes=['lugar', 'area', 'tipovivi',
'energcocinar',
'sanitario', 'pared', 'piso',
'abastagua'],
subsets={'electricity': ['public',
'planpri',
'noelec',
'coopele']})) 
)
 return my_features<data_type_conversions>
def run(fold,model):
df = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv')
 df_test = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv')
 df["kfold"] = -1
kf = model_selection.StratifiedKFold(n_splits=4, shuffle=True, random_state=42)
 for f,(train_idx, val_idx)in enumerate(kf.split(X=df, y=df.SalePrice.values)) :
df.loc[val_idx, 'kfold'] = f
df = feature_engineering(df)
 df_test = feature_engineering(df_test)
 df_test.fillna(0,inplace=True)
 df.SalePrice = np.log1p(df.SalePrice)
 df = pd.concat([df,df_test],axis=0)
 df = pd.get_dummies(df)
 for feature in df.columns:
all_value_counts = df[feature].value_counts()
zero_value_counts = all_value_counts.iloc[0]
if zero_value_counts / len(df)> 0.99:
df.drop(feature,axis=1,inplace=True)
 df_test = df.loc[df["Id"].between(1461,2919)]
df = df.loc[df["Id"].between(1,1460)]
df_train = df[df.kfold != fold].reset_index(drop=True)
 df_valid = df[df.kfold == fold].reset_index(drop=True)
 numeric_features = [feature for feature in df_train.columns if feature not in("Id", "kfold","SalePrice")]
x_train = df_train[numeric_features].values
x_valid = df_valid[numeric_features].values
reg = MODELS[model]
reg.fit(x_train,df_train.SalePrice.values)
 valid_preds = reg.predict(x_valid)
 test_preds = reg.predict(df_test[numeric_features].values)
 rmse = np.sqrt(mean_squared_error(df_valid.SalePrice.values, valid_preds))
mae = mean_absolute_error(df_valid.SalePrice.values, valid_preds)
 print(f"FOLD={fold}, MODEL = {model}, RMSE = {rmse}, MAE ={mae} ")
 return(test_preds)
X = preprocess(df)
 y = df.groupby('idhogar' ).Target.mean().astype(int)
<choose_model_class>
preds_df = pd.DataFrame()
for fold in range(3):
for keys,items in MODELS.items() :
preds_df["fold"+str(fold)+keys] = run(fold,keys)
clf = lgb.LGBMClassifier(class_weight='balanced', boosting_type='dart',
drop_rate=0.9, min_data_in_leaf=100,
max_bin=255,
n_estimators=500,
bagging_fraction=0.01,
min_sum_hessian_in_leaf=1,
importance_type='gain',
learning_rate=0.1,
max_depth=-1,
num_leaves=31 )<load_from_csv>
def run(fold,model):
df = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv')
 df_test = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv')
 df["kfold"] = -1
kf = model_selection.StratifiedKFold(n_splits=4, shuffle=True, random_state=42)
 for f,(train_idx, val_idx)in enumerate(kf.split(X=df, y=df.SalePrice.values)) :
df.loc[val_idx, 'kfold'] = f
df = feature_engineering(df)
 df_test = feature_engineering(df_test)
 df_test.fillna(0,inplace=True)
 df.SalePrice = np.log1p(df.SalePrice)
 df = pd.concat([df,df_test],axis=0)
 df = pd.get_dummies(df)
 for feature in df.columns:
all_value_counts = df[feature].value_counts()
zero_value_counts = all_value_counts.iloc[0]
if zero_value_counts / len(df)> 0.99:
df.drop(feature,axis=1,inplace=True)
 df_test = df.loc[df["Id"].between(1461,2919)]
df = df.loc[df["Id"].between(1,1460)]
df_train = df[df.kfold != fold].reset_index(drop=True)
 df_valid = df[df.kfold == fold].reset_index(drop=True)
 numeric_features = [feature for feature in df_train.columns if feature not in("Id", "kfold","SalePrice")]
x_train = df_train[numeric_features].values
x_valid = df_valid[numeric_features].values
reg = MODELS_stack
reg.fit(x_train,df_train.SalePrice.values)
 valid_preds = reg.predict(x_valid)
 test_preds = reg.predict(df_test[numeric_features].values)
 rmse = np.sqrt(mean_squared_error(df_valid.SalePrice.values, valid_preds))
mae = mean_absolute_error(df_valid.SalePrice.values, valid_preds)
 print(f"FOLD={fold}, MODEL = {model}, RMSE = {rmse}, MAE ={mae} ")
 return(test_preds)
 for fold in range(3):
preds_df["fold"+str(fold)+"stack"] = run(fold,keys )
df_test = pd.read_csv(".. /input/test.csv" ).set_index('Id')
 X_test = preprocess(df_test )<find_best_model_class>
for cols in preds_df.columns:
preds_df[cols] = np.expm1(preds_df[cols])
kf = StratifiedKFold(n_splits=5, shuffle=True)
 predicts = []
for train_index, test_index in kf.split(X, y):
print("
X_train, X_val = X.iloc[train_index], X.iloc[test_index]
y_train, y_val = y.iloc[train_index], y.iloc[test_index]
clf.fit(X_train, y_train, eval_set=[(X_val, y_val)],
early_stopping_rounds=20)
 predicts.append(clf.predict(X_test))<create_dataframe>
for e,col in enumerate(preds_df.columns):
if e == 0:
preds_df['SalePrice'] = preds_df[col]
elif col in ['fold0stack','fold1stack','fold2stack']:
preds_df['SalePrice'] += preds_df[col]*4
else:
preds_df['SalePrice'] += preds_df[col]
predict_by_hh = pd.DataFrame(np.array(predicts ).mean(axis=0 ).round().astype(int),
columns=['Target'],
index=X_test.index )<data_type_conversions>
preds_df['SalePrice'] = preds_df['SalePrice']/30
predict = df_test.join(predict_by_hh, on='idhogar')[['Target']].astype(int )<save_to_csv>
df_test = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv',usecols=["Id"] )
predict.to_csv("output.csv" )<set_options>
preds_df["id"] = df_test.values.flatten()
%matplotlib inline 
<compute_train_metric>
submission = preds_df[['id','SalePrice']]
<load_from_csv><EOS> 
submission.to_csv(".. /.. /kaggle/working/submission.csv", index=False )