Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<feature_engineering>
warnings.simplefilter(action='ignore', category=FutureWarning)
 warnings.simplefilter("ignore", category=ConvergenceWarning)
 pd.pandas.set_option('display.max_columns', None)
 pd.set_option('display.float_format', lambda x: '%.3f' % x )
train['dependency'] = np.sqrt(train['SQBdependency'])
 test['dependency'] = np.sqrt(test['SQBdependency'])
 train.loc[train['edjefa'] == "no", "edjefa"] = 0
train.loc[train['edjefa'] == "yes", "edjefa"] = 1
train['edjefa'] = train['edjefa'].astype("int")
 train.loc[train['edjefe'] == "no", "edjefe"] = 0
train.loc[train['edjefe'] == "yes", "edjefe"] = 1
train['edjefe'] = train['edjefe'].astype("int")
 test.loc[test['edjefa'] == "no", "edjefa"] = 0
test.loc[test['edjefa'] == "yes", "edjefa"] = 4
test['edjefa'] = test['edjefa'].astype("int")
 test.loc[test['edjefe'] == "no", "edjefe"] = 0
test.loc[test['edjefe'] == "yes", "edjefe"] = 4
test['edjefe'] = test['edjefe'].astype("int")
 train['edjef'] = np.max(train[['edjefa','edjefe']], axis=1)
 test['edjef'] = np.max(test[['edjefa','edjefe']], axis=1)
 train['v2a1']=train['v2a1'].fillna(0)
 test['v2a1']=test['v2a1'].fillna(0)
 test['v18q1']=test['v18q1'].fillna(0)
 train['v18q1']=train['v18q1'].fillna(0)
 train['rez_esc']=train['rez_esc'].fillna(0)
 test['rez_esc']=test['rez_esc'].fillna(0)
 train.loc[train.meaneduc.isnull() , "meaneduc"] = 0
train.loc[train.SQBmeaned.isnull() , "SQBmeaned"] = 0
test.loc[test.meaneduc.isnull() , "meaneduc"] = 0
test.loc[test.SQBmeaned.isnull() , "SQBmeaned"] = 0
train.loc[(train.v14a == 1)&(train.sanitario1 == 1)&(train.abastaguano == 0), "v14a"] = 0
train.loc[(train.v14a == 1)&(train.sanitario1 == 1)&(train.abastaguano == 0), "sanitario1"] = 0
test.loc[(test.v14a == 1)&(test.sanitario1 == 1)&(test.abastaguano == 0), "v14a"] = 0
test.loc[(test.v14a == 1)&(test.sanitario1 == 1)&(test.abastaguano == 0), "sanitario1"] = 0<filter>
def data() :
train_ = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/train.csv")
 test_ = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/test.csv")
 dataframe = pd.concat([train_, test_], ignore_index=True)
 return dataframe, train_, test_
df, train, test = data()
df.head()
d={}
weird=[]
for row in train.iterrows() :
idhogar=row[1]['idhogar']
target=row[1]['Target']
if idhogar in d:
if d[idhogar]!=target:
weird.append(idhogar)
 else:
d[idhogar]=target
for i in set(weird):
hhold=train[train['idhogar']==i][['idhogar', 'parentesco1', 'Target']]
target=hhold[hhold['parentesco1']==1]['Target'].tolist() [0]
for row in hhold.iterrows() :
idx=row[0]
if row[1]['parentesco1']!=1:
train.at[idx, 'Target']=target<count_unique_values>
df.isnull().any().sum()
print("Unique Households:", train.idhogar.nunique())
 print("Total Rows:", len(train))<categorify>
cat_cols = [col for col in df.columns if df[col].dtypes == 'O']
print('Kategorik Değişken Sayısı: ', len(cat_cols))
def cat_summary(data, categorical_cols, target, number_of_classes=10):
var_count = 0
vars_more_classes = []
for var in categorical_cols:
if len(df[var].value_counts())<= number_of_classes:
print(pd.DataFrame({var: data[var].value_counts() ,
"Ratio": 100 * data[var].value_counts() / len(data),
"TARGET_MEDIAN": data.groupby(var)[target].median() }), end="
")
 var_count += 1
else:
vars_more_classes.append(data[var].name)
 print('%d categorical variables have been described' % var_count, end="
")
 print('There are', len(vars_more_classes), "variables have more than", number_of_classes, "classes", end="
")
 print('Variable names have more than %d classes:' % number_of_classes, end="
")
 print(vars_more_classes)
 cat_summary(df, cat_cols, "SalePrice" )
train['max_esc'] = train.groupby('idhogar')['escolari'].transform('max')
 train['min_esc'] = train.groupby('idhogar')['escolari'].transform('min')
 test['max_esc'] = test.groupby('idhogar')['escolari'].transform('max')
 test['min_esc'] = test.groupby('idhogar')['escolari'].transform('min' )<categorify>
for col in ['Neighborhood', 'Exterior1st', 'Exterior2nd']:
print(df[col].value_counts() )
train['max_rez'] = train.groupby('idhogar')['rez_esc'].transform('max')
 train['min_rez'] = train.groupby('idhogar')['rez_esc'].transform('min')
 train['mean_rez'] = train.groupby('idhogar')['rez_esc'].transform('mean')
 test['max_rez'] = test.groupby('idhogar')['rez_esc'].transform('max')
 test['min_rez'] = test.groupby('idhogar')['rez_esc'].transform('min')
 test['mean_rez'] = test.groupby('idhogar')['rez_esc'].transform('mean')
 train['num_over_18'] = 0
train['num_over_18'] = train[train.age >= 18].groupby('idhogar' ).transform("count")
 train['num_over_18'] = train.groupby("idhogar")["num_over_18"].transform("max")
 train['num_over_18'] = train['num_over_18'].fillna(0)
 test['num_over_18'] = 0
test['num_over_18'] = test[test.age >= 18].groupby('idhogar' ).transform("count")
 test['num_over_18'] = test.groupby("idhogar")["num_over_18"].transform("max")
 test['num_over_18'] = test['num_over_18'].fillna(0 )<feature_engineering>
def find_correlation(dataframe, corr_limit=0.60):
high_correlations = []
low_correlations = []
for col in num_cols:
if col == "SalePrice":
pass
else:
correlation = dataframe[[col, "SalePrice"]].corr().loc[col, "SalePrice"]
print(col, correlation)
 if abs(correlation)> corr_limit:
high_correlations.append(col + ": " + str(correlation))
else:
low_correlations.append(col + ": " + str(correlation))
return low_correlations, high_correlations
low_corrs, high_corrs = find_correlation(df )
def extract_features(df):
df['bedrooms_to_rooms'] = df['bedrooms']/df['rooms']
df['rent_to_rooms'] = df['v2a1']/df['rooms']
df['tamhog_to_rooms'] = df['tamhog']/df['rooms']
df['r4t3_to_tamhog'] = df['r4t3']/df['tamhog']
df['r4t3_to_rooms'] = df['r4t3']/df['rooms']
df['v2a1_to_r4t3'] = df['v2a1']/df['r4t3']
df['v2a1_to_r4t3'] = df['v2a1']/(df['r4t3'] - df['r4t1'])
 df['hhsize_to_rooms'] = df['hhsize']/df['rooms']
df['rent_to_hhsize'] = df['v2a1']/df['hhsize']
df['rent_to_over_18'] = df['v2a1']/df['num_over_18']
df.loc[df.num_over_18 == 0, "rent_to_over_18"] = df[df.num_over_18 == 0].v2a1
df['dependency_yes'] = df['dependency'].apply(lambda x: 1 if x == 'yes' else 0)
 df['dependency_no'] = df['dependency'].apply(lambda x: 1 if x == 'no' else 0)
 def do_features(df):
feats_div = [('children_fraction', 'r4t1', 'r4t3'), 
('working_man_fraction', 'r4h2', 'r4t3'), 
('all_man_fraction', 'r4h3', 'r4t3'), 
('human_density', 'tamviv', 'rooms'), 
('human_bed_density', 'tamviv', 'bedrooms'), 
('rent_per_person', 'v2a1', 'r4t3'), 
('rent_per_room', 'v2a1', 'rooms'), 
('mobile_density', 'qmobilephone', 'r4t3'), 
('tablet_density', 'v18q1', 'r4t3'), 
('mobile_adult_density', 'qmobilephone', 'r4t2'), 
('tablet_adult_density', 'v18q1', 'r4t2'),
]
feats_sub = [('people_not_living', 'tamhog', 'tamviv'), 
('people_weird_stat', 'tamhog', 'r4t3')]
for f_new, f1, f2 in feats_div:
df['fe_' + f_new] =(df[f1] / df[f2] ).astype(np.float32)
 for f_new, f1, f2 in feats_sub:
df['fe_' + f_new] =(df[f1] - df[f2] ).astype(np.float32)
 aggs_num = {'age': ['min', 'max', 'mean'],
'escolari': ['min', 'max', 'mean']
}
aggs_cat = {'dis': ['mean']}
for s_ in ['estadocivil', 'parentesco', 'instlevel']:
for f_ in [f_ for f_ in df.columns if f_.startswith(s_)]:
aggs_cat[f_] = ['mean', 'count']
for name_, df_ in [('18', df.query('age >= 18')) ]:
df_agg = df_.groupby('idhogar' ).agg({**aggs_num, **aggs_cat} ).astype(np.float32)
 df_agg.columns = pd.Index(['agg' + name_ + '_' + e[0] + "_" + e[1].upper() for e in df_agg.columns.tolist() ])
 df = df.join(df_agg, how='left', on='idhogar')
 del df_agg
return df
def convert_OHE2LE(df):
tmp_df = df.copy(deep=True)
 for s_ in ['pared', 'piso', 'techo', 'abastagua', 'sanitario', 'energcocinar', 'elimbasu',
'epared', 'etecho', 'eviv', 'estadocivil', 'parentesco',
'instlevel', 'lugar', 'tipovivi',
'manual_elec']:
if 'manual_' not in s_:
cols_s_ = [f_ for f_ in df.columns if f_.startswith(s_)]
elif 'elec' in s_:
cols_s_ = ['public', 'planpri', 'noelec', 'coopele']
sum_ohe = tmp_df[cols_s_].sum(axis=1 ).unique()
if 0 in sum_ohe:
print('The OHE in {} is incomplete.A new column will be added before label encoding' 
.format(s_))
col_dummy = s_+'_dummy'
tmp_df[col_dummy] =(tmp_df[cols_s_].sum(axis=1)== 0 ).astype(np.int8)
 cols_s_.append(col_dummy)
 sum_ohe = tmp_df[cols_s_].sum(axis=1 ).unique()
if 0 in sum_ohe:
print("The category completion did not work")
 tmp_cat = tmp_df[cols_s_].idxmax(axis=1)
 tmp_df[s_ + '_LE'] = LabelEncoder().fit_transform(tmp_cat ).astype(np.int16)
 if 'parentesco1' in cols_s_:
cols_s_.remove('parentesco1')
 tmp_df.drop(cols_s_, axis=1, inplace=True)
 return tmp_df
def process_df(df_):
for f_ in ['v2a1', 'v18q1', 'meaneduc', 'SQBmeaned']:
df_[f_] = df_[f_].fillna(0)
 df_['rez_esc'] = df_['rez_esc'].fillna(-1)
 return do_features(df_ )<create_dataframe>
def rare_analyser(dataframe, target, rare_perc):
rare_columns = [col for col in df.columns if len(df[col].value_counts())<= 20
and(df[col].value_counts() / len(df)< rare_perc ).any(axis=None)]
for var in rare_columns:
print(var, ":", len(dataframe[var].value_counts()))
print(pd.DataFrame({"COUNT": dataframe[var].value_counts() ,
"RATIO": dataframe[var].value_counts() / len(dataframe),
"TARGET_MEDIAN": dataframe.groupby(var)[target].median() }), end="
")
 rare_analyser(df, "SalePrice", 0.01)
 def rare_encoder(dataframe, rare_perc):
temp_df = dataframe.copy()
rare_columns = [col for col in temp_df.columns if temp_df[col].dtypes == 'O'
and(temp_df[col].value_counts() / len(temp_df)< rare_perc ).any(axis=None)]
for var in rare_columns:
tmp = temp_df[var].value_counts() / len(temp_df)
 rare_labels = tmp[tmp < rare_perc].index
temp_df[var] = np.where(temp_df[var].isin(rare_labels), 'Rare', temp_df[var])
 return temp_df
df = rare_encoder(df, 0.01)
 rare_analyser(df, "SalePrice", 0.01 )
extract_features(train)
 extract_features(test)
 train = process_df(train)
 test = process_df(test )<categorify>
drop_list = ["Street", "Utilities", "LandSlope", "PoolQC", "MiscFeature"]
cat_cols = [col for col in df.columns if df[col].dtypes == 'O'
and col not in drop_list]
for col in drop_list:
df.drop(col, axis=1, inplace=True)
 rare_analyser(df, "SalePrice", 0.01 )
cols_2_ohe = ['eviv_LE', 'etecho_LE', 'epared_LE', 'elimbasu_LE',
'energcocinar_LE', 'sanitario_LE', 'manual_elec_LE',
'pared_LE']
cols_nums = ['age', 'meaneduc', 'dependency',
'hogar_nin', 'hogar_adul', 'hogar_mayor', 'hogar_total',
'bedrooms', 'overcrowding']
def convert_geo2aggs(df_):
tmp_df = pd.concat([df_[(['lugar_LE', 'idhogar']+cols_nums)],
pd.get_dummies(df_[cols_2_ohe],
columns=cols_2_ohe)],axis=1)
 geo_agg = tmp_df.groupby(['lugar_LE','idhogar'] ).mean().groupby('lugar_LE' ).mean().astype(np.float32)
 geo_agg.columns = pd.Index(['geo_' + e for e in geo_agg.columns.tolist() ])
 del tmp_df
return df_.join(geo_agg, how='left', on='lugar_LE')
<concatenate>
def one_hot_encoder(dataframe, categorical_cols, nan_as_category=True):
original_columns = list(dataframe.columns)
 dataframe = pd.get_dummies(dataframe, columns=categorical_cols, dummy_na=nan_as_category, drop_first=True)
 new_columns = [c for c in dataframe.columns if c not in original_columns]
return dataframe, new_columns
df, new_cols_ohe = one_hot_encoder(df, cat_cols)
 cat_summary(df, new_cols_ohe, "SalePrice" )
def train_test_apply_func(train_, test_, func_):
test_['Target'] = 0
xx = pd.concat([train_, test_])
 xx_func = func_(xx)
 train_ = xx_func.iloc[:train_.shape[0], :]
test_ = xx_func.iloc[train_.shape[0]:, :].drop('Target', axis=1)
 del xx, xx_func
return train_, test_<feature_engineering>
def missing_values_table(dataframe):
variables_with_na = [col for col in dataframe.columns if dataframe[col].isnull().sum() > 0]
n_miss = dataframe[variables_with_na].isnull().sum().sort_values(ascending=False)
 ratio =(dataframe[variables_with_na].isnull().sum() / dataframe.shape[0] * 100 ).sort_values(ascending=False)
 missing_df = pd.concat([n_miss, np.round(ratio, 2)], axis=1, keys=['n_miss', 'ratio'])
 print(missing_df)
 return variables_with_na
missing_values_table(df)
train, test = train_test_apply_func(train, test, convert_OHE2LE )<prepare_x_and_y>
df = df.apply(lambda x: x.fillna(x.median()), axis=0)
 missing_values_table(df )
train2 = train.copy()
target = train2.Target
train2 = train2.drop("Target", axis=1)
 train_hhs = train2.idhogar
households = train2.idhogar.unique()
cv_hhs = np.random.choice(households, size=int(len(households)* 0.25), replace=False)
 cv_idx = np.isin(train2.idhogar, cv_hhs)
 X_cv = train2[cv_idx]
y_cv = target[cv_idx]
X_tr = train2[~cv_idx]
y_tr = target[~cv_idx]
X_tr = train2
y_tr = target<drop_column>
def outlier_thresholds(dataframe, variable):
quartile1 = dataframe[variable].quantile(0.05)
 quartile3 = dataframe[variable].quantile(0.95)
 interquantile_range = quartile3 - quartile1
up_limit = quartile3 + 1.5 * interquantile_range
low_limit = quartile1 - 1.5 * interquantile_range
return low_limit, up_limit
def has_outliers(dataframe, num_col_names, plot=False):
variable_names = []
for col in num_col_names:
low_limit, up_limit = outlier_thresholds(dataframe, col)
 if dataframe[(dataframe[col] > up_limit)|(dataframe[col] < low_limit)].any(axis=None):
number_of_outliers = dataframe[(dataframe[col] > up_limit)|(dataframe[col] < low_limit)].shape[0]
print(col, ":", number_of_outliers)
 variable_names.append(col)
 if plot:
sns.boxplot(x=dataframe[col])
 plt.show()
return variable_names
has_outliers(df, num_cols )
drop_cols = train.columns[train.isnull().any() ]
X_tr.drop(drop_cols, axis=1, inplace=True)
 X_cv.drop(drop_cols, axis=1, inplace=True)
 test.drop(drop_cols, axis=1, inplace=True )<define_variables>
def replace_with_thresholds(dataframe, variable):
low_limit, up_limit = outlier_thresholds(dataframe, variable)
 dataframe.loc[(dataframe[variable] < low_limit), variable] = low_limit
dataframe.loc[(dataframe[variable] > up_limit), variable] = up_limit
for col in num_cols:
replace_with_thresholds(df, col)
 has_outliers(df, num_cols )
extra_drop_cols = []<define_variables>
df['TotalSF'] = df['TotalBsmtSF'] + df['1stFlrSF'] + df['2ndFlrSF']
df["OverallGrade"] = df["OverallQual"] * df["OverallCond"]
df['haspool'] = df['PoolArea'].apply(lambda x: 1 if x > 0 else 0)
 df['has2ndfloor'] = df['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)
 df.loc[df["OverallQual"] < 2, "OverallQual"] = 2
df.loc[df["GarageCars"] > 4, "GarageCars"] = 4
df.loc[(df["OverallQual"] == 2)|(df["OverallQual"] == 3), "OverallQual"] = 3
df.loc[df["GarageYrBlt"] == 2207, "GarageYrBlt"] = 2005
df.loc[df["Fireplaces"] == 4, "Fireplaces"] = 0
df.loc[df["Fireplaces"] == 3, "Fireplaces"] = 2
df["new_area"] = df["GrLivArea"] + df["GarageArea"]
df["new_home"] = df["YearBuilt"]
df.loc[df["new_home"] == df["YearRemodAdd"], "new_home"] = 0
df.loc[df["new_home"] != df["YearRemodAdd"], "new_home"] = 1
df["new_bath"] = df["FullBath"] +(df["HalfBath"] * 0.5)
 df.head()
extra_drop_cols = ['fe_people_weird_stat',
'min_rez',
'dependency_no',
'rez_esc',
'parentesco1',
'parentesco_LE',
'dependency_yes']<train_model>
missing_values_table(df)
 has_outliers(df, num_cols )
et_drop_cols = ['Id', 'idhogar'] + extra_drop_cols
et = ExtraTreesClassifier(n_estimators=250, max_depth=24, min_impurity_decrease=1e-10, min_samples_leaf=1, min_samples_split=2,
min_weight_fraction_leaf=0.0, n_jobs=-1, random_state=1, verbose=1)
 et.fit(X_tr.drop(et_drop_cols, axis=1), y_tr )<compute_train_metric>
train_df = df[df['SalePrice'].notnull() ]
test_df = df[df['SalePrice'].isnull() ]
X = train_df.drop('SalePrice', axis=1)
 y = train_df[["SalePrice"]]
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.20, random_state=46)
 models = [('LinearRegression', LinearRegression()), 
('Ridge', Ridge()), 
('Lasso', Lasso()), 
('ElasticNet', ElasticNet())]
results = []
names = []
for name, model in models:
model.fit(X_train, y_train)
 y_pred = model.predict(X_test)
 result = np.sqrt(mean_squared_error(y_test, y_pred))
results.append(result)
 names.append(name)
 msg = "%s: %f" %(name, result)
 print(msg)
 df["SalePrice"].mean()
<compute_test_metric><EOS> 
X = df.loc[:1459, :].drop(["SalePrice", "Id"], axis=1)
 y = df.loc[:1459, "SalePrice"]
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.20, random_state=46)
 catboost_model = CatBoostRegressor()
catboost_model = catboost_model.fit(X_train, y_train)
 y_pred = catboost_model.predict(X_test)
 print(np.sqrt(mean_squared_error(y_pred, y_test)))
 df1 = pd.DataFrame({"Id": df.loc[1460:, "Id"],
"SalePrice": catboost_model.predict(df.loc[1460:, :].drop(["SalePrice", "Id"], axis=1)) })
 df1.to_csv('submission_catboost.csv', index=False )