Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<define_variables>
sns.set(rc={'figure.figsize':(12,8.27)})
 for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
validation_scores = {}<compute_test_metric>
def train_models(X,y,kfolds=12):
kfold = KFold(kfolds)
 for model in models:
cf_result = cross_val_score(model['regressor'], X, y, cv=kfold,scoring='neg_mean_squared_log_error')
 model['cv_results'] = np.sqrt(cf_result*-1)
 msg = f"Regressor: {model['name']}, rmsle:{cf_result.mean().round(11)} "
print(msg)
 class AveragingModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, models):
self.models = models
def fit(self, X, y):
self.models_ = [clone(x)for x in self.models]
for model in self.models_:
model.fit(X, y)
 return self
def predict(self, X):
predictions = np.column_stack([
model.predict(X)for model in self.models_
])
 return np.mean(predictions, axis=1 )
scorer = make_scorer(f1_score, greater_is_better=True, average = 'macro' )<choose_model_class>
models = []
models.append({'name':'XBR','regressor':XGBRegressor(random_state=0)})
 models.append({'name' :'Random Forest','regressor': RandomForestRegressor(criterion='mse',max_depth=35,max_features='sqrt',n_estimators=150,random_state=0)})
 models.append({'name': 'Ridge','regressor' :Ridge(alpha=2.0,copy_X=True,fit_intercept=False,max_iter=1000,normalize=True,random_state=0)})
 models.append({'name': 'AveragingModels','regressor' :AveragingModels(models =(XGBRegressor() ,Lasso() , Ridge(alpha=2.0), RandomForestRegressor(n_estimators=200,random_state=0)))} )
skf = StratifiedKFold(n_splits=5 )<choose_model_class>
original_train = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 original_test = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv')
 y_column = 'SalePrice'
original_train.head()
lightgbm = lgb.LGBMClassifier(class_weight='balanced', boosting_type='dart',
drop_rate=0.9, min_data_in_leaf=100,
max_bin=255,
n_estimators=500,
bagging_fraction=0.01,
min_sum_hessian_in_leaf=1,
importance_type='gain',
learning_rate=0.1,
max_depth=-1,
num_leaves=31)
<train_model>
train = original_train.copy().drop(columns=['Id'])
 test = original_test.copy()
predicts_lgb = []
for train_index, test_index in skf.split(X_train, Y_train):
X_t, X_v = X_train.iloc[train_index], X_train.iloc[test_index]
y_t, y_v = Y_train.iloc[train_index], Y_train.iloc[test_index]
lightgbm.fit(X_t, y_t, eval_set=[(X_v, y_v)], early_stopping_rounds=50)
 predicts_lgb.append(lightgbm.predict(X_test))<save_to_csv>
train.isnull().sum().sort_values(ascending=False )
lightgbm_pred = np.array(predicts_lgb ).mean(axis=0 ).round().astype(int)
 submission_lgb = pd.DataFrame({'Id': test_id,
'Target': lightgbm_pred})
 submission_lgb.to_csv('submissionLGB.csv', index=False )<train_model>
no_missing_train = train.copy().drop(columns=['PoolQC','MiscFeature','Alley','Fence'])
 no_missing_train.FireplaceQu = no_missing_train.FireplaceQu.fillna('NA')
 no_missing_train.BsmtQual = no_missing_train.BsmtQual.fillna('NA')
 no_missing_train.BsmtFinType1 = no_missing_train.BsmtFinType1.fillna('NA')
 no_missing_train.BsmtCond = no_missing_train.BsmtCond.fillna('NA')
 no_missing_train.LotFrontage = no_missing_train.LotFrontage.mean()
for column in no_missing_train.isnull().sum().sort_values(ascending=False ).index:
no_missing_train[column].fillna(no_missing_train[column].mode() [0],inplace=True)
 no_missing_test = test.copy().drop(columns=['PoolQC','MiscFeature','Alley','Fence'])
 no_missing_test.FireplaceQu = no_missing_test.FireplaceQu.fillna('NA')
 no_missing_test.BsmtQual = no_missing_test.BsmtQual.fillna('NA')
 no_missing_test.BsmtFinType1 = no_missing_test.BsmtFinType1.fillna('NA')
 no_missing_test.BsmtCond = no_missing_test.BsmtCond.fillna('NA')
 no_missing_test.LotFrontage = no_missing_test.LotFrontage.mean()
for column in no_missing_test.isnull().sum().sort_values(ascending=False ).index:
no_missing_test[column].fillna(no_missing_test[column].mode() [0],inplace=True )
predicts_xgb = []
for train_index, test_index in skf.split(X_train, Y_train):
X_t, X_v = X_train.iloc[train_index], X_train.iloc[test_index]
y_t, y_v = Y_train.iloc[train_index], Y_train.iloc[test_index]
xgboost.fit(X_t, y_t, eval_set=[(X_v, y_v)], early_stopping_rounds=50)
 predicts_xgb.append(xgboost.predict(X_test))<save_to_csv>
no_missing_test.isnull().sum().sort_values(ascending=False )
xgboost_pred = np.array(predicts_xgb ).mean(axis=0 ).round().astype(int)
 submission_xgb = pd.DataFrame({'Id': test_id,
'Target': xgboost_pred})
 submission_xgb.to_csv('submissionXGB.csv', index=False )<choose_model_class>
train_eng = no_missing_train.copy()
test_eng = no_missing_test.copy()
train_eng['TotalBath'] = train_eng.FullBath +(train_eng.HalfBath * 0.5)
 train_eng.drop(columns=['FullBath','HalfBath'],inplace=True)
 test_eng['TotalBath'] = test_eng.FullBath +(test_eng.HalfBath * 0.5)
 test_eng.drop(columns=['FullBath','HalfBath'],inplace=True)
submission_model_lgb_old = lgb.LGBMClassifier(class_weight='balanced', boosting_type='dart',
drop_rate=0.9, min_data_in_leaf=100,
max_bin=255,
n_estimators=500,
bagging_fraction=0.01,
min_sum_hessian_in_leaf=1,
importance_type='gain',
learning_rate=0.1,
max_depth=-1,
num_leaves=31 )<train_model>
X_train_eng = train_eng.drop(columns=[y_column] ).select_dtypes([int,float,bool])
 y_train_eng = train_eng[y_column]
X_test_eng = test_eng.drop(columns=['Id'] ).select_dtypes([int,float,bool])
submission_model_lgb_old.fit(X_train, Y_train);<predict_on_test>
train_featured = train_eng.copy()
test_featured = test_eng.copy()
train_featured["SalePrice"] = np.log1p(train_featured["SalePrice"])
 qualities = {'NA':0,'Po':1,'Fa':2,'TA':3,'Gd':4,'Ex':5}
train_featured.BsmtQual.replace(qualities,inplace=True)
 train_featured.BsmtCond.replace(qualities,inplace=True)
 train_featured.HeatingQC.replace(qualities,inplace=True)
 train_featured.FireplaceQu.replace(qualities,inplace=True)
 train_featured.GarageQual.replace(qualities,inplace=True)
 train_featured.GarageCond.replace(qualities,inplace=True)
 train_featured.KitchenQual.replace(qualities,inplace=True)
 train_featured.ExterQual.replace(qualities,inplace=True)
 train_featured.ExterCond.replace(qualities,inplace=True)
 test_featured.BsmtQual.replace(qualities,inplace=True)
 test_featured.BsmtCond.replace(qualities,inplace=True)
 test_featured.HeatingQC.replace(qualities,inplace=True)
 test_featured.FireplaceQu.replace(qualities,inplace=True)
 test_featured.GarageQual.replace(qualities,inplace=True)
 test_featured.GarageCond.replace(qualities,inplace=True)
 test_featured.KitchenQual.replace(qualities,inplace=True)
 test_featured.ExterQual.replace(qualities,inplace=True)
 test_featured.ExterCond.replace(qualities,inplace=True)
 y_n = {'N':0,'Y':1}
train_featured.CentralAir.replace(y_n,inplace=True)
 test_featured.CentralAir.replace(y_n,inplace=True )
final_pred_lgb_old = submission_model_lgb_old.predict(X_test)
 final_pred_lgb_old = final_pred_lgb_old.astype(int )<save_to_csv>
X_train_feat = train_featured.drop(columns=[y_column] ).select_dtypes([int,float,bool])
 y_train_feat = train_featured[y_column]
X_test_feat = test_featured.drop(columns=['Id'] ).select_dtypes([int,float,bool])
 print("Training model for feature transformation technique")
 train_models(X_train_feat,y_train_feat )
submission_lgb_old = pd.DataFrame({'Id': test_id,
'Target': final_pred_lgb_old})
 submission_lgb_old.to_csv('submissionLGBold.csv', index=False )<save_to_csv>
train_rm = train_featured.copy()
test_rm = test_featured.copy()
<save_to_csv>
X_train_rm = train_rm.drop(columns=[y_column] ).select_dtypes([int,float,bool])
 y_train_rm = train_rm[y_column]
X_test_rm = test_rm.drop(columns=['Id'] ).select_dtypes([int,float,bool])
 print("Training model for feature removal technique")
 train_models(X_train_rm,y_train_rm )
<import_modules>
train_out = train_rm.copy()
test_out = test_rm.copy()
train_out = train_out.drop(train_out[train_out.LotArea > 50000].index)
 train_out = train_out.drop(train_out[train_out.GrLivArea > 4000].index )
%matplotlib inline
print(os.listdir(".. /input")) 
<categorify>
X_train_out = train_out.drop(columns=[y_column] ).select_dtypes([int,float,bool])
 y_train_out = train_out[y_column]
X_test_out = test_out.drop(columns=['Id'] ).select_dtypes([int,float,bool])
 print("Training model for outliers removal technique")
 train_models(X_train_out,y_train_out )
def dprint(*args, **kwargs):
print("[{}] ".format(datetime.datetime.now().strftime("%Y-%m-%d %H:%M")) + \
" ".join(map(str,args)) , **kwargs)
 id_name = 'Id'
target_name = 'Target'<load_from_csv>
train_encoded = train_out.copy()
test_encoded = test_out.copy()
train_encoded[train_encoded.select_dtypes('object' ).columns] = train_encoded[train_encoded.select_dtypes('object' ).columns].astype('category')
 test_encoded[test_encoded.select_dtypes('object' ).columns] = test_encoded[test_encoded.select_dtypes('object' ).columns].astype('category')
 category_columns = train_encoded.select_dtypes('category' ).columns
encoder = ce.CatBoostEncoder(cols=category_columns)
 encoder.fit(train_encoded[category_columns],train_encoded['SalePrice'])
 train_encoded[category_columns] = encoder.transform(train_encoded[category_columns])
 test_encoded[category_columns] = encoder.transform(test_encoded[category_columns])
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )<concatenate>
X_train_encoded = train_encoded.drop(columns=[y_column] ).select_dtypes([int,float,bool])
 y_train_encoded = train_encoded[y_column]
X_test_encoded = test_encoded.drop(columns=['Id'] ).select_dtypes([int,float,bool])
 print("Training model for encoded features technique")
 train_models(X_train_encoded,y_train_encoded)
train['is_test'] = 0
test['is_test'] = 1
df_all = pd.concat([train, test], axis=0 )<data_type_conversions>
estimator = XGBRegressor(random_state=0)
 selector = RFE(estimator, n_features_to_select=25, step=1)
 selector = selector.fit(X_train_encoded, y_train_encoded)
 best_columns = X_train_encoded.columns[selector.support_]
train_models(X_train_encoded[best_columns],y_train_encoded )
dprint('Clean features...')
 cols = ['dependency']
for c in tqdm(cols):
x = df_all[c].values
strs = []
for i, v in enumerate(x):
try:
val = float(v)
 except:
strs.append(v)
 val = np.nan
x[i] = val
strs = np.unique(strs)
 for s in strs:
df_all[c + '_' + s] = df_all[c].apply(lambda x: 1 if x == s else 0)
 df_all[c] = x
df_all[c] = df_all[c].astype(float)
 dprint("Done." )<feature_engineering>
parameters ={"alpha" : [0.05, 0.10, 0.50, 0.75, 1, 2,4,5 ] }
rdg = Ridge()
clf = GridSearchCV(rdg, parameters)
 clf.fit(X_train_encoded,y_train_encoded )
dprint("Extracting features...")
 def extract_features(df):
df['bedrooms_to_rooms'] = df['bedrooms']/df['rooms']
df['rent_to_rooms'] = df['v2a1']/df['rooms']
df['rent_to_bedrooms'] = df['v2a1']/df['bedrooms']
df['tamhog_to_rooms'] = df['tamhog']/df['rooms']
df['tamhog_to_bedrooms'] = df['tamhog']/df['bedrooms']
df['r4t3_to_tamhog'] = df['r4t3']/df['tamhog']
df['r4t3_to_rooms'] = df['r4t3']/df['rooms']
df['r4t3_to_bedrooms'] = df['r4t3']/df['bedrooms']
df['rent_to_r4t3'] = df['v2a1']/df['r4t3']
df['v2a1_to_r4t3'] = df['v2a1']/(df['r4t3'] - df['r4t1'])
 df['hhsize_to_rooms'] = df['hhsize']/df['rooms']
df['hhsize_to_bedrooms'] = df['hhsize']/df['bedrooms']
df['rent_to_hhsize'] = df['v2a1']/df['hhsize']
df['qmobilephone_to_r4t3'] = df['qmobilephone']/df['r4t3']
df['qmobilephone_to_v18q1'] = df['qmobilephone']/df['v18q1']
extract_features(train)
 extract_features(test)
 dprint("Done." )<categorify>
clf.best_params_
def encode_data(df):
yes_no_map = {'no': 0, 'yes': 1}
df['dependency'] = df['dependency'].replace(yes_no_map ).astype(np.float32)
 df['edjefe'] = df['edjefe'].replace(yes_no_map ).astype(np.float32)
 df['edjefa'] = df['edjefa'].replace(yes_no_map ).astype(np.float32)
 df['idhogar'] = LabelEncoder().fit_transform(df['idhogar'] )<categorify>
best_model = AveragingModels(models =(Ridge(alpha=2.0,copy_X=True,fit_intercept=False,max_iter=1000,normalize=True,random_state=0), RandomForestRegressor(criterion='mse',max_depth=35,max_features='sqrt',n_estimators=150,random_state=0)))
 best_model.fit(X_train_encoded,y_train_encoded )
dprint("Encoding Data.... ")
 encode_data(train)
 encode_data(test)
 dprint("Done..." )<feature_engineering>
predicted = test[['Id']].copy()
predicted['SalePrice'] = np.expm1(best_model.predict(X_test_encoded))
def do_features(df):
feats_div = [('children_fraction', 'r4t1', 'r4t3'), 
('working_man_fraction', 'r4h2', 'r4t3'), 
('all_man_fraction', 'r4h3', 'r4t3'), 
('human_density', 'tamviv', 'rooms'), 
('human_bed_density', 'tamviv', 'bedrooms'), 
('rent_per_person', 'v2a1', 'r4t3'), 
('rent_per_room', 'v2a1', 'rooms'), 
('mobile_density', 'qmobilephone', 'r4t3'), 
('tablet_density', 'v18q1', 'r4t3'), 
('mobile_adult_density', 'qmobilephone', 'r4t2'), 
('tablet_adult_density', 'v18q1', 'r4t2'),
]
feats_sub = [('people_not_living', 'tamhog', 'tamviv'), 
('people_weird_stat', 'tamhog', 'r4t3')]
for f_new, f1, f2 in feats_div:
df['fe_' + f_new] =(df[f1] / df[f2] ).astype(np.float32)
 for f_new, f1, f2 in feats_sub:
df['fe_' + f_new] =(df[f1] - df[f2] ).astype(np.float32)
 aggs_num = {'age': ['min', 'max', 'mean'],
'escolari': ['min', 'max', 'mean']
}
aggs_cat = {'dis': ['mean']}
for s_ in ['estadocivil', 'parentesco', 'instlevel']:
for f_ in [f_ for f_ in df.columns if f_.startswith(s_)]:
aggs_cat[f_] = ['mean', 'count']
for name_, df_ in [('18', df.query('age >= 18')) ]:
df_agg = df_.groupby('idhogar' ).agg({**aggs_num, **aggs_cat} ).astype(np.float32)
 df_agg.columns = pd.Index(['agg' + name_ + '_' + e[0] + "_" + e[1].upper() for e in df_agg.columns.tolist() ])
 df = df.join(df_agg, how='left', on='idhogar')
 del df_agg
df.drop([f_ for f_ in df.columns if f_.startswith('SQB')or f_ == 'agesq'], axis=1, inplace=True)
 df.drop(['Id', 'idhogar'], axis=1, inplace=True)
 df.drop(['hhsize', 'female', 'area2'], axis=1, inplace=True)
 return df<feature_engineering>
predicted.to_csv('HousePricingEduardoRenz.csv',index=False )