Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<define_variables>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from scipy.stats import norm
house_head_feats = ['head_school_years', 'head_edu_level'] + head_gender_feats<categorify>
train = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv')
 test = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv' )
data_all = add_quality(data_all, componente='pared', component='wall')
 data_all = add_quality(data_all, componente='techo', component='roof')
 data_all = add_quality(data_all, componente='viv', component='floor')
 print(data_all.shape )<categorify>
train_id = train['Id']
test_id = test['Id']
train = train.drop('Id', axis=1)
 test = test.drop('Id', axis=1 )
translate = {'blolad': 'block',
'zocalo': 'socket',
'preb': 'cement',
'des': 'waste',
'mad': 'wood',
'zinc': 'zink',
'fibras': 'natural_fibers',
'other': 'other'}
data_all = to_english(data_all, sp_pre='pared', eng_pre='wall_',
translate=translate)
 wall_feats = [cc for cc in data_all.columns if 'wall_' in cc]
translate = {
'moscer': 'mosaic',
'cemento': 'cement',
'other': 'other',
'natur': 'natural',
'notiene': 'no_floor',
'madera': 'wood'
}
data_all = to_english(data_all, sp_pre='piso', eng_pre='floor_', translate=translate)
 floor_feats = [cc for cc in data_all.columns if 'floor_' in cc]
translate = {
'zinc': 'zinc',
'entrepiso': 'fiber cement',
'cane': 'natural fibers',
'otro': 'other'
}
data_all = to_english(data_all, sp_pre='techo', eng_pre='roof_', translate=translate)
 roof_feats = [cc for cc in data_all.columns if 'roof_' in cc]
material_feats = roof_feats + wall_feats + floor_feats<categorify>
to_drop = ['PoolQC', 'MiscFeature', 'Alley', 'FireplaceQu']
train = train.drop(to_drop, axis=1)
 test = test.drop(to_drop, axis=1 )
translate = {
'guadentro': 'inside_house',
'guafuera': 'outside_house',
'guano': 'no'
}
data_all = to_english(data_all, sp_pre='abasta', eng_pre='water_provision_',
translate=translate)
 water_feats = [cc for cc in data_all.columns if 'water_provision_' in cc]
translate = {
'public': 'public',
'planpri': 'private_plan',
'noelec': 'no',
'coopele': 'cooperate'
}
data_all = to_english(data_all, sp_pre='', eng_pre='electric_', translate=translate)
 elec_feats = [cc for cc in data_all.columns if 'electric_' in cc]
translate = {
'cinar1': 'no',
'cinar2': 'electricity',
'cinar3': 'gas',
'cinar4': 'charcoal'
}
data_all = to_english(data_all, sp_pre='energco', eng_pre='energy_', translate=translate)
 energy_feats = [cc for cc in data_all.columns if 'energy_' in cc]
translate = {
'1': 'no',
'2': 'sewer',
'3': 'septic_tank',
'5': 'black hole',
'6': 'other'
}
data_all = to_english(data_all, sp_pre='sanitario', eng_pre='toilet_', translate=translate)
 toilet_feats = [cc for cc in data_all.columns if 'toilet_' in cc]
translate = {
'1': 'tanker truck',
'2': 'buried',
'3': 'burning',
'4': 'throw empty place',
'5': 'throw to river',
'6': 'other'
}
data_all = to_english(data_all, sp_pre='elimbasu', eng_pre='rubbish_', translate=translate)
 rubbish_feats = [cc for cc in data_all.columns if 'rubbish_' in cc]
facility_feats = water_feats + elec_feats + energy_feats + toilet_feats + rubbish_feats<categorify>
LOG_FEATURES = ['LotFrontage', 'LotArea', 'BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF', 'TotalBsmtSF', '1stFlrSF', '2ndFlrSF', 'LowQualFinSF', 'GrLivArea', 'BsmtFullBath', 'BsmtHalfBath', 'FullBath', 'HalfBath', 'BedroomAbvGr', 'KitchenAbvGr', 'TotRmsAbvGrd', 'Fireplaces', 'GarageCars', 'GarageArea', 'WoodDeckSF', 'OpenPorchSF', 'PoolArea']
def engineer_features(data):
data['MSSubClass'] = data['MSSubClass'].astype(str)
 data['YrSold'] = data['YrSold'].astype(str)
 data['MoSold'] = data['MoSold'].astype(str)
 data['haspool'] = data['PoolArea'].apply(lambda x: 1 if x > 0 else 0)
 data['has2ndfloor'] = data['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)
 data['hasgarage'] = data['GarageArea'].apply(lambda x: 1 if x > 0 else 0)
 data['hasbsmt'] = data['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 data['hasfireplace'] = data['Fireplaces'].apply(lambda x: 1 if x > 0 else 0)
 for feature in LOG_FEATURES:
data[feature] = np.log1p(data[feature])
 return data
translate = {
'1': 'own_fully_paid',
'2': 'own_pay_installment',
'3': 'rented',
'4': 'precarious',
'5': 'other'
}
data_all = to_english(data_all, sp_pre='tipovivi', eng_pre='living_type_',
translate=translate)
 live_feats = [cc for cc in data_all.columns if 'living_type_' in cc]<split>
train = engineer_features(train)
 test = engineer_features(test )
head_df = data_all.query('parentesco1 == 1')
 train = head_df.loc[head_df['Target'].notnull() , :]
test = head_df.loc[head_df['Target'].isnull() , :]<define_variables>
from sklearn.model_selection import train_test_split, GridSearchCV, RandomizedSearchCV, cross_val_score
from sklearn.pipeline import Pipeline
from sklearn.compose import ColumnTransformer, make_column_selector, TransformedTargetRegressor
from sklearn.preprocessing import OneHotEncoder, RobustScaler, OrdinalEncoder, StandardScaler
from sklearn.linear_model import Ridge, Lasso, RidgeCV
from sklearn.neighbors import KNeighborsRegressor
from sklearn.kernel_ridge import KernelRidge
from sklearn.impute import SimpleImputer
from sklearn.ensemble import RandomForestRegressor, StackingRegressor
from sklearn.metrics import mean_squared_error, mean_squared_log_error, make_scorer
import xgboost
import lightgbm as lgb
features = basic_feats + house_head_feats + material_feats + facility_feats + live_feats + fe_feats + agg_feats
print('<split>
X_train, y_train = train.drop('SalePrice', axis=1), train['SalePrice']
X, y = train[['idhogar', 'Id'] + features], train['Target']
X_train, X_valid, y_train, y_valid = train_test_split(X[features], y,
test_size=0.1, random_state=0 )<define_search_space>
categorical = [i for i in X_train.columns if train.dtypes[i] == 'object']
numerical = [i for i in X_train.columns if train.dtypes[i] != 'object']
gbm = lgb.LGBMClassifier(n_jobs=4, random_state=0, class_weight='balanced')
 param_grid = {'num_leaves': np.arange(10, 50, 10),
'learning_rate': np.arange(0.05, 0.2, 0.05),
'n_estimators': np.arange(10, 50, 5)}
scoring = {'f1_macro': get_scorer('f1_macro')}
metric = 'f1_macro'<train_on_grid>
def col_indicies(names):

 return np.isin(missing_columns_output, names)
 ordinal = ['ExterQual', 'ExterCond', 'BsmtQual', 'BsmtCond', 'HeatingQC', 'KitchenQual', 'GarageQual', 'GarageCond']
non_ordinal_categorical = np.setdiff1d(categorical, ordinal)
 housing_encoder = ColumnTransformer([ 
('quality', OrdinalEncoder(categories=[['NA', 'Po', 'Fa', 'TA', 'Gd', 'Ex']]*len(ordinal)) , col_indicies(ordinal)) , 
('LandSlope', OrdinalEncoder(categories=[['Gtl', 'Mod', 'Sev']]), col_indicies(['LandSlope'])) , 
('BsmtExposure', OrdinalEncoder(categories=[['NA', 'No', 'Mn', 'Av', 'Gd']]), col_indicies(['BsmtExposure'])) , 
('Functional', OrdinalEncoder(categories=[['Sal', 'Sev', 'Maj2', 'Maj1', 'Mod', 'Min2', 'Min1', 'Typ']]), col_indicies(['Functional'])) , 
('GarageFinish', OrdinalEncoder(categories=[['NA', 'Unf', 'RFn', 'Fin']]), col_indicies(['GarageFinish'])) , 
('Fence', OrdinalEncoder(categories=[['NA', 'MnWw', 'GdWo', 'MnPrv', 'GdPrv']]), col_indicies(['Fence'])) , 
('BsmtFinType', OrdinalEncoder(categories=[['NA', 'Unf', 'LwQ', 'Rec', 'BLQ', 'ALQ', 'GLQ']]*2), col_indicies(['BsmtFinType1', 'BsmtFinType2'])) , 
('PavedDrive', OrdinalEncoder(categories=[['N', 'P', 'Y']]), col_indicies(['PavedDrive'])) , 
('categorical', OneHotEncoder(handle_unknown='ignore'), col_indicies(non_ordinal_categorical))
], remainder='passthrough')
 housing_all_hot_encoder = ColumnTransformer([ 
('categorical', OneHotEncoder(handle_unknown='ignore'), col_indicies(categorical))
], remainder='passthrough' )
gs = GridSearchCV(gbm,
param_grid=param_grid,
scoring=scoring,
cv=5,
refit=metric,
verbose=True,
n_jobs=4, 
)
 gs.fit(X_train, y_train)
 best_estimator = gs.best_estimator_<predict_on_test>
ridge = Pipeline([ 
('fill_missing', missing_preprocessor), 
('encoding', housing_encoder), 
('scale', RobustScaler(with_centering=False)) , 
('ridge', Ridge(alpha=11.905772393787833))
])
 ridge = TransformedTargetRegressor(ridge, func=np.log1p, inverse_func=np.expm1)
 hot_ridge = Pipeline([ 
('fill_missing', missing_preprocessor), 
('encoding', housing_all_hot_encoder), 
('scale', RobustScaler(with_centering=False)) , 
('ridge', Ridge(alpha=11.905772393787833))
])
 hot_ridge = TransformedTargetRegressor(hot_ridge, func=np.log1p, inverse_func=np.expm1)
y_pred = best_estimator.predict(X_valid)
 macro_f1 = f1_score(y_valid, y_pred, average='macro')
 print(macro_f1 )<predict_on_test>
X_test = test[['Id', 'idhogar'] + features]
y_pred = best_estimator.predict(X_test[features])
 pred_for_heads = pd.DataFrame({'Id': X_test['Id'], 'idhogar': X_test['idhogar'],
'Target': y_pred})
 pred_for_heads['Target'] = pred_for_heads['Target'].apply(int )<merge>
lasso = Pipeline([ 
('fill_missing', missing_preprocessor), 
('encoding', housing_encoder), 
('scale', RobustScaler(with_centering=False)) , 
('lasso', Lasso(alpha=0.000565620644760902))
])
 lasso = TransformedTargetRegressor(lasso, func=np.log1p, inverse_func=np.expm1)
 hot_lasso = Pipeline([ 
('fill_missing', missing_preprocessor), 
('encoding', housing_all_hot_encoder), 
('scale', RobustScaler(with_centering=False)) , 
('lasso', Lasso(alpha=0.000565620644760902))
])
 hot_lasso = TransformedTargetRegressor(hot_lasso, func=np.log1p, inverse_func=np.expm1 )
test_all = data_all.loc[data_all['Target'].isnull() , ['Id', 'idhogar']]
submit = pd.merge(test_all, pred_for_heads[['idhogar', 'Target']], how='left' )<count_missing_values>
sum(submit['Target'].isnull() )<data_type_conversions>
kernel_ridge = Pipeline([ 
('fill_missing', missing_preprocessor), 
('encoding', housing_encoder), 
('scale', RobustScaler(with_centering=False)) , 
('kridge', KernelRidge(kernel='poly', degree=2, alpha=48.93900918477494))
])
 kernel_ridge = TransformedTargetRegressor(kernel_ridge, func=np.log1p, inverse_func=np.expm1 )
submit.fillna(1, inplace=True)
 submit['Target'] = submit['Target'].astype(np.int16)
 submit.head()<save_to_csv>
submit[['Id', 'Target']].to_csv('submisssion.csv', index=False )<import_modules>
knn = Pipeline([ 
('fill_missing', missing_preprocessor), 
('encoding', housing_encoder), 
('scale', RobustScaler(with_centering=False)) , 
('knn', KNeighborsRegressor(n_neighbors=9, weights='distance', p=1))
])
 knn = TransformedTargetRegressor(knn, func=np.log1p, inverse_func=np.expm1)
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.preprocessing import LabelEncoder, OneHotEncoder, StandardScaler
from sklearn.model_selection import StratifiedKFold
from sklearn.feature_selection import chi2, mutual_info_classif
from sklearn.decomposition import PCA
from sklearn.manifold import TSNE
from sklearn.utils.class_weight import compute_class_weight, compute_sample_weight
import xgboost as xgb
import lightgbm as lgb
from catboost import CatBoostClassifier<categorify>
sc = StandardScaler()
le = LabelEncoder()
onehot = OneHotEncoder(sparse=False )<load_from_csv>
rf = Pipeline([ 
('fill_missing', missing_preprocessor), 
('encoding', housing_encoder), 
('scale', RobustScaler(with_centering=False)) , 
('rf', RandomForestRegressor(n_jobs=-1))
])
 rf = TransformedTargetRegressor(rf, func=np.log1p, inverse_func=np.expm1)
train_df = pd.read_csv('.. /input/train.csv')
 train_df.head()<load_from_csv>
param_grid = {
'regressor__rf__max_depth': [10, 20, 30, 40, 50, 60, None],
'regressor__rf__max_features': ['auto', 'sqrt'],
'regressor__rf__min_samples_leaf': [1, 2, 4],
'regressor__rf__n_estimators': [200, 400, 600, 800, 1000, 1200]
}
tuned_rf = RandomizedSearchCV(rf, param_grid, n_jobs=-1, cv=3, n_iter=10)
test_df = pd.read_csv('.. /input/test.csv')
 test_df.head()<concatenate>
param_init = {
"max_depth": 5,
"n_estimators": 3000,
"learning_rate": 0.01,
"subsample": 0.5,
"colsample_bytree": 0.7,
"min_child_weight": 1.5,
"reg_alpha": 0.75,
"reg_lambda": 0.4,
"seed": 42,
}
param_fit = {
"xgb__eval_metric": "rmse",
"xgb__verbose": 200
}
xgb_model = Pipeline([ 
('fill_missing', missing_preprocessor), 
('encoding', housing_encoder), 
('scale', RobustScaler(with_centering=False)) , 
('xgb', xgboost.XGBRegressor(**param_init))
])
 xgb_model = TransformedTargetRegressor(xgb_model, func=np.log1p, inverse_func=np.expm1 )
all_df = train_df.append(test_df, sort=False)
 all_df.shape<drop_column>
param_init = {
'objective': 'regression',
'num_leaves': 5,
'learning_rate': 0.05,
'n_estimators': 720,
'max_bin': 55,
'bagging_fraction': 0.8,
'bagging_freq': 5,
'feature_fraction': 0.2319,
'feature_fraction_seed': 9,
'bagging_seed': 9,
'min_data_in_leaf': 6,
'min_sum_hessian_in_leaf': 11
}
lgbm_model = Pipeline([ 
('fill_missing', missing_preprocessor), 
('encoding', housing_encoder), 
('scale', RobustScaler(with_centering=False)) , 
('lgbm', lgb.LGBMRegressor(**param_init))
])
 lgbm_model = TransformedTargetRegressor(lgbm_model, func=np.log1p, inverse_func=np.expm1 )
all_df.drop(['rez_esc', 'v18q1', 'v2a1'], axis=1, inplace=True)
 all_df.fillna({'SQBmeaned': all_df['SQBmeaned'].median() , 'meaneduc': all_df['meaneduc'].median() }, inplace=True )<define_variables>
models = {
'ridge': ridge,
'lasso': lasso,
'kernel ridge': kernel_ridge,
'knn': knn,
'random forest': rf,
'xgboost': xgb_model,
'lgbm': lgbm_model
}
float_cols = [col for col in all_df.columns if all_df[col].dtype=='float64']
int_cols = [col for col in all_df.columns if all_df[col].dtype=='int64']
object_cols = [col for col in all_df.columns if all_df[col].dtype=='object']<prepare_x_and_y>
for name, model in models.items() :
print(f'{name}:')
 print(np.sqrt(np.mean(cross_val_score(model, X_train, y_train, cv=4, scoring=make_scorer(mean_squared_log_error), n_jobs=-1))))
log_meaneduc = np.log1p(all_df['meaneduc'])
 log_overcrowding = np.log1p(all_df['overcrowding'])
 log_SQBovercrowding = np.log1p(all_df['SQBovercrowding'])
 log_SQBdependency = np.log1p(all_df['SQBdependency'])
 log_SQBmeaned = np.log1p(all_df['SQBmeaned'])
 temp_df = pd.DataFrame({'log_meaneduc': log_meaneduc, 'log_overcrowding': log_overcrowding, 'log_SQBovercrowding': log_SQBovercrowding, 'log_SQBdependency': log_SQBdependency, 'log_SQBmeaned': log_SQBmeaned})
 temp_df.head()<drop_column>
estimators = [('lasso', lasso), 
('ridge', ridge), 
('xgb', xgb_model), 
('lgbm', lgbm_model)]
stacking_regressor = StackingRegressor(estimators=estimators, final_estimator=RidgeCV())
 print(np.sqrt(np.mean(cross_val_score(stacking_regressor, X_train, y_train, cv=3, scoring=make_scorer(mean_squared_log_error), n_jobs=-1))))
all_df[['meaneduc', 'overcrowding', 'SQBovercrowding', 'SQBdependency', 'SQBmeaned']] = temp_df[['log_meaneduc', 'log_overcrowding', 'log_SQBovercrowding', 'log_SQBdependency', 'log_SQBmeaned']]<categorify>
best_model = stacking_regressor
_ = best_model.fit(train.drop('SalePrice', axis=1), train['SalePrice'] )
<drop_column><EOS> 
sub = pd.DataFrame()
sub['Id'] = test_id
sub['SalePrice'] = best_model.predict(test)
 sub.to_csv('submission.csv', index=False )