Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<define_variables>
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib
import matplotlib.pyplot as plt
from scipy.stats import skew
from scipy.stats.stats import pearsonr
from sklearn import metrics
from sklearn.metrics import mean_squared_error
cat_columns = [col for col in installments_payments.columns if installments_payments[col].dtype == 'object']<categorify>
train = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/train.csv")
 test = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/test.csv" )
installments_payments = pd.get_dummies(installments_payments,cat_columns, dummy_na = True )<feature_engineering>
all_data = pd.concat(( train.loc[:,'MSSubClass':'SaleCondition'],test.loc[:,'MSSubClass':'SaleCondition']))
installments_payments['AMOUNT_DIFF'] = installments_payments['AMT_INSTALMENT'] - installments_payments['AMT_PAYMENT'] 
<feature_engineering>
train["SalePrice"] = np.log1p(train["SalePrice"])
 numeric_feats = all_data.dtypes[all_data.dtypes != "object"].index
installments_payments['AMOUNT_PERC'] = installments_payments['AMT_PAYMENT']/installments_payments['AMT_INSTALMENT']<feature_engineering>
skewed_feats = train[numeric_feats].apply(lambda x: skew(x.dropna()))
skewed_feats = skewed_feats[skewed_feats > 0.75]
skewed_feats = skewed_feats.index
skewed_feats
all_data[skewed_feats] = np.log1p(all_data[skewed_feats] )
installments_payments['DAYS_P'] = installments_payments['DAYS_ENTRY_PAYMENT']-installments_payments['DAYS_INSTALMENT']
installments_payments['DAYS_I'] = installments_payments['DAYS_INSTALMENT']-installments_payments['DAYS_ENTRY_PAYMENT']<define_variables>
all_data = pd.get_dummies(all_data)
 all_data = all_data.fillna(all_data.mean() )
aggregations = {
'NUM_INSTALMENT_VERSION': ['nunique'],
'DAYS_P': ['max', 'mean', 'sum'],
'DAYS_I': ['max', 'mean', 'sum'],
'AMOUNT_DIFF': ['max', 'mean', 'sum', 'var'],
'AMOUNT_PERC': ['max', 'mean', 'sum', 'var'],
'AMT_INSTALMENT': ['max', 'mean', 'sum'],
'AMT_PAYMENT': ['min', 'max', 'mean', 'sum'],
'DAYS_ENTRY_PAYMENT': ['max', 'mean', 'sum']
}<groupby>
X_train = all_data[:train.shape[0]]
X_test = all_data[train.shape[0]:]
y = train.SalePrice
installments_payments_agg = installments_payments.groupby('SK_ID_CURR' ).agg(aggregations )<rename_columns>
def rmse_cv(model):
rmse= np.sqrt(-cross_val_score(model, X_train, y, scoring="neg_mean_squared_error", cv = 5))
return(rmse )
installments_payments_agg.columns = pd.Index(['INSTALL_' + e[0] + "_" + e[1].upper() for e in installments_payments_agg.columns.tolist() ])
 installments_payments_agg.reset_index(inplace=True)
<merge>
model_ridge = Ridge()
installments_payments = installments_payments.merge(installments_payments_agg, on = 'SK_ID_CURR',how = 'left' )<feature_engineering>
alphas = [0.05, 0.1, 0.3, 1, 3, 5, 10, 15, 30, 50, 75]
cv_ridge = [rmse_cv(Ridge(alpha = alpha)).mean()
for alpha in alphas]
previous_application['DAYS_FIRST_DRAWING'].replace(365243, np.nan, inplace= True)
 previous_application['DAYS_FIRST_DUE'].replace(365243, np.nan, inplace= True)
 previous_application['DAYS_LAST_DUE_1ST_VERSION'].replace(365243, np.nan, inplace= True)
 previous_application['DAYS_LAST_DUE'].replace(365243, np.nan, inplace= True)
 previous_application['DAYS_TERMINATION'].replace(365243, np.nan, inplace= True)
 previous_application['INTEREST_PERC'] =(previous_application['RATE_INTEREST_PRIMARY']/100)*previous_application['AMT_DOWN_PAYMENT']
previous_application['INTEREST_ANN_PERC'] =(previous_application['RATE_INTEREST_PRIMARY']/100)*previous_application['AMT_ANNUITY']
previous_application['INTEREST_CREDIT_PERC'] =(previous_application['RATE_INTEREST_PRIMARY']/100)*previous_application['AMT_CREDIT']
previous_application['FIRST_LAST'] = previous_application['DAYS_FIRST_DUE'] - previous_application['DAYS_LAST_DUE'] 
<feature_engineering>
model_lasso = LassoCV(alphas = [1, 0.1, 0.001, 0.0005] ).fit(X_train, y)
 rmse_cv(model_lasso ).mean()
previous_application['APPLICATION_ACTUAL_CREDIT'] = previous_application['AMT_APPLICATION']/previous_application['AMT_CREDIT']<define_variables>
imp_coef = pd.concat([coef.sort_values().head(10),
coef.sort_values().tail(10)] )
num_aggregations = {
'AMT_ANNUITY': ['min', 'max', 'mean'],
'AMT_APPLICATION': ['min', 'max', 'mean'],
'AMT_CREDIT': ['min', 'max', 'mean'],
'INTEREST_CREDIT_PERC': ['min', 'max', 'mean', 'var'],
'AMT_DOWN_PAYMENT': ['min', 'max', 'mean'],
'AMT_GOODS_PRICE': ['min', 'max', 'mean'],
'HOUR_APPR_PROCESS_START': ['min', 'max', 'mean'],
'RATE_DOWN_PAYMENT': ['min', 'max', 'mean'],
'DAYS_DECISION': ['min', 'max', 'mean'],
'CNT_PAYMENT': ['mean', 'sum'],
'FIRST_LAST': ['mean', 'max', 'min']
}
prev_agg = previous_application.groupby('SK_ID_CURR' ).agg(num_aggregations)
 prev_agg.columns = pd.Index(['PREV_' + e[0] + "_" + e[1].upper() for e in prev_agg.columns.tolist() ])
 prev_agg.reset_index(inplace=True)
 previous_application = previous_application.merge(prev_agg, on = 'SK_ID_CURR', how = 'left')
 del prev_agg<filter>
import xgboost as xgb
approved = previous_application[previous_application['NAME_CONTRACT_STATUS'] == 'Approved']<groupby>
dtrain = xgb.DMatrix(X_train, label = y)
 dtest = xgb.DMatrix(X_test)
 params = {"max_depth":2, "eta":0.1}
model = xgb.cv(params, dtrain, num_boost_round=500, early_stopping_rounds=100 )
approved_agg = approved.groupby('SK_ID_CURR' ).agg(num_aggregations )<rename_columns>
model_xgb = xgb.XGBRegressor(n_estimators=360, max_depth=2, learning_rate=0.1)
 model_xgb.fit(X_train, y )
approved_agg.columns = pd.Index(['APPROVED_' + e[0] + "_" + e[1].upper() for e in approved_agg.columns.tolist() ] )<merge>
xgb_preds = np.expm1(model_xgb.predict(X_test))
lasso_preds = np.expm1(model_lasso.predict(X_test))
previous_application = previous_application.merge(approved_agg, how='left', on='SK_ID_CURR' )<merge>
predictions = pd.DataFrame({"xgb":xgb_preds, "lasso":lasso_preds})
 predictions.plot(x = "xgb", y = "lasso", kind = "scatter" )
refused = previous_application[previous_application['NAME_CONTRACT_STATUS'] == 'Refused']
refused_agg = refused.groupby('SK_ID_CURR' ).agg(num_aggregations)
 refused_agg.columns = pd.Index(['REFUSED_' + e[0] + "_" + e[1].upper() for e in refused_agg.columns.tolist() ])
 refused_agg.reset_index(inplace=True)
 previous_application = previous_application.merge(refused_agg, how='left', on='SK_ID_CURR')
<define_variables>
from catboost import Pool
from catboost import CatBoostRegressor
from sklearn.model_selection import train_test_split
import numpy as np
aggregations = {
'MONTHS_BALANCE': ['max', 'mean', 'size'],
'SK_DPD': ['max', 'mean'],
'SK_DPD_DEF': ['max', 'mean']
} 
<groupby>
float_cols = X.dtypes[X.dtypes == "float"].index
X.loc[:,float_cols] = X[float_cols].astype(str)
 X_train.shape, X.shape
POS_CASH_AGG = POS_CASH_balance.groupby('SK_ID_CURR' ).agg(aggregations )<rename_columns>
X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.25, random_state=SEED)
 cat_features = list(range(X.shape[1]))
POS_CASH_AGG.columns = pd.Index(['POS_CASH_' + e[0] + "_" + e[1].upper() for e in POS_CASH_AGG.columns.tolist() ] )<groupby>
train_data = Pool(data=X_train,
label=y_train,
cat_features=cat_features 
)
 valid_data = Pool(data=X_valid,
label=y_valid,
cat_features=cat_features
)
POS_CASH_AGG['COUNT'] = POS_CASH_AGG.groupby('SK_ID_CURR' ).size()<categorify>
%%time
params = { 'early_stopping_rounds': 100,
'verbose': False,
'random_seed': SEED,
'eval_metric':'RMSE'
}
cbc_7 = CatBoostRegressor(**params)
 cbc_7.fit(train_data,
eval_set=valid_data,
use_best_model=True,
plot=True 
);
cat_columns = [col for col in POS_CASH_balance.columns if POS_CASH_balance[col].dtype == 'object']
POS_CASH_balance = pd.get_dummies(POS_CASH_balance,cat_columns, dummy_na = True)
 POS_CASH_balance = POS_CASH_balance.merge(POS_CASH_AGG, how = 'left', on = 'SK_ID_CURR')
 POS_CASH_balance.head()<groupby>
cbc_7.get_feature_importance(prettified=True )
POS_CASH_balance = POS_CASH_balance.groupby('SK_ID_CURR' ).mean().reset_index()<drop_column>
X_test = test_df
float_cols_test = X_test.dtypes[X_test.dtypes == "float"].index
X_test.loc[:,float_cols_test] = X_test[float_cols_test].astype(str)
 X_train.shape, X.shape , X_test.shape
del POS_CASH_AGG, POS_CASH_balance['SK_ID_PREV']<categorify>
%%time
n_fold = 3
folds = StratifiedKFold(n_splits=n_fold, shuffle=True, random_state=SEED)
 params = {
'early_stopping_rounds': 100,
'verbose': False,
'random_seed': SEED,
'eval_metric':'RMSE'
}
test_data = Pool(data=X_test,
cat_features=cat_features)
 bins = np.linspace(0, 1, 100)
 y_binned = np.digitize(y, bins)
 scores = []
catboost_prediction = np.zeros(X_test.shape[0])
 for fold_n,(train_index, valid_index)in enumerate(folds.split(X, y_binned)) :
X_train, X_valid = X.iloc[train_index], X.iloc[valid_index]
y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]
train_data = Pool(data=X_train,
label=y_train,
cat_features=cat_features)
 valid_data = Pool(data=X_valid,
label=y_valid,
cat_features=cat_features)
 model = CatBoostRegressor(**params)
 model.fit(train_data,
eval_set=valid_data,
use_best_model=True 
)
 score = model.get_best_score() ['validation']['RMSE']
scores.append(score)
 y_pred = model.predict(test_data)
 catboost_prediction += y_pred
catboost_prediction /= n_fold
print('CV mean: {:.4f}, CV std: {:.4f}'.format(np.mean(scores), np.std(scores)))
 print('RMSE:', np.sqrt(metrics.mean_squared_error(y_valid, model.predict(X_valid))))
y = data['TARGET']
del data['TARGET']
categorical_features = [col for col in data.columns if data[col].dtype == 'object']
one_hot_df = pd.concat([data,test])
 one_hot_df = pd.get_dummies(one_hot_df, columns=categorical_features)
 data = one_hot_df.iloc[:data.shape[0],:]
test = one_hot_df.iloc[data.shape[0]:,]
print(data.shape, test.shape )<filter>
h20_preds = pd.read_csv('.. /input/house-price-predictions-h20-automl-without-tuning/house_sales_price_pred_full.csv')
 predictions = pd.DataFrame({"h20":h20_preds['SalePrice'], "lasso":lasso_preds})
 predictions.plot(x = "h20", y = "lasso", kind = "scatter")
 predictions = pd.DataFrame({"h20":h20_preds['SalePrice'], "cb":cb_preds})
 predictions.plot(x = "h20", y = "cb", kind = "scatter")
 predictions = pd.DataFrame({"h20":h20_preds['SalePrice'], "xgb":xgb_preds})
 predictions.plot(x = "h20", y = "xgb", kind = "scatter" )
print('Removing features with more than 80% missing...')
 test = test[test.columns[data.isnull().mean() < 0.80]]
data = data[data.columns[data.isnull().mean() < 0.80]]
print(data.shape, test.shape )<define_variables>
preds = 0.57*lasso_preds + 0.0*xgb_preds + 0.0*cb_preds + 0.43*h20_preds['SalePrice']
cb_solution = pd.DataFrame({"id":test.Id, "SalePrice":preds})
 cb_solution.to_csv("house_price_preds.csv", index = False )