Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<merge>
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib
import matplotlib.pyplot as plt
from scipy.stats import skew,norm
from scipy.stats.stats import pearsonr
tmp = previous[previous['AMT_CREDIT'] > 0]
for i in ['Cash loans','Consumer loans','Revolving loans']:
for df in [train,test]:
tmp1 = tmp[tmp['NAME_CONTRACT_TYPE'] == i].groupby(['SK_ID_CURR'])['AMT_CREDIT'].count().reset_index()
tmp1.columns = ['SK_ID_CURR','des']
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp1, on=['SK_ID_CURR'], how='left')
 df['count_' + "_".join(i.lower().split())] = tmp_merge['des']<merge>
train = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/train.csv")
 test = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/test.csv")
 train_size = train.shape[0]
submission = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/sample_submission.csv")
 warnings.filterwarnings(action="ignore" )
tmp = previous[previous['AMT_CREDIT'].isnull() ]
for i in ['Cash loans','Consumer loans','Revolving loans']:
for df in [train,test]:
tmp1 = tmp[tmp['NAME_CONTRACT_TYPE'] == i].groupby(['SK_ID_CURR'])['SK_ID_PREV'].count().reset_index()
tmp1.columns = ['SK_ID_CURR','des']
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp1, on=['SK_ID_CURR'], how='left')
 df['count_null_' + "_".join(i.lower().split())] = tmp_merge['des']<groupby>
all_data = pd.concat(( train.loc[:,'MSSubClass':'SaleCondition'],
test.loc[:,'MSSubClass':'SaleCondition']))
train["SalePrice"] = np.log1p(train["SalePrice"])
 numeric_features = all_data.dtypes[all_data.dtypes != "object"].index
skewed_features = train[numeric_features].apply(lambda x: skew(x.dropna()))
skewed_features = skewed_features[skewed_features > 0.75]
skewed_features = skewed_features.index
all_data[skewed_features] = np.log1p(all_data[skewed_features])
 all_data = pd.get_dummies(all_data)
 all_data = all_data.fillna(all_data.mean() )
tmp = previous[previous['AMT_CREDIT'] > 0].groupby(['SK_ID_CURR'])['DAYS_DECISION']\ 
.agg({"returns": [np.min, np.max,np.mean]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_day_decision'] = tmp_merge['des1']
df['max_day_decision'] = tmp_merge['des2']
df['mean_day_decision'] = tmp_merge['des3']<groupby>
X_train = all_data[:train_size]
X_test = all_data[train_size:]
y_train = train.SalePrice
tmp = previous[previous['NAME_CONTRACT_STATUS'] != 'Approved'].groupby(['SK_ID_CURR'])['DAYS_DECISION']\ 
.agg({"returns": [np.min, np.max,np.mean]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_day_decision_fail'] = tmp_merge['des1']
df['max_day_decision_fail'] = tmp_merge['des2']
df['mean_day_decision_fail'] = tmp_merge['des3']<merge>
def mean_absolute_percentage_error(y_true, y_pred):
y_true, y_pred = np.array(y_true), np.array(y_pred)
 return np.mean(np.abs(( y_true - y_pred)/ y_true)) * 100
tmp = previous[(~previous['NAME_CASH_LOAN_PURPOSE'].isin(['XAP','XNA'])) ]
for df in [train,test]:
tmp1 = tmp.groupby(['SK_ID_CURR'])['SK_ID_PREV'].count().reset_index()
tmp1.columns = ['SK_ID_CURR','des']
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp1, on=['SK_ID_CURR'], how='left')
 df['count_clear_reason'] = tmp_merge['des']<merge>
alphas = [0.05, 0.1, 0.5, 1, 5, 10, 20, 40, 80]
cv_ridge = [np.sqrt(-cross_val_score(Ridge(alpha = alpha), X_train, y_train, scoring="neg_mean_squared_error", cv = 5)).mean()
for alpha in alphas]
best_alpha = alphas[cv_ridge.index(min(cv_ridge)) ]
cv_ridge = pd.Series(cv_ridge, index = alphas)
 min_rmse = cv_ridge.min()
print("best alpha : ", best_alpha, " and minimal rmse : ", min_rmse )
tmp = previous.groupby(['SK_ID_CURR'])['DAYS_TERMINATION']\ 
.agg({"returns": [np.min, np.max,np.mean]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_day_termination'] = tmp_merge['des1']
df['max_day_termination'] = tmp_merge['des2']
df['mean_day_termination'] = tmp_merge['des3']<merge>
model_Ridge = Ridge(10 ).fit(X_train, y_train)
 ridge_predict = np.exp(model_Ridge.predict(X_test))
solution = pd.DataFrame({"id":test.Id, "SalePrice": ridge_predict})
 percentage_error = mean_absolute_percentage_error(submission.SalePrice, solution.SalePrice)
 print(percentage_error )
tmp = previous.groupby(['SK_ID_CURR'])['DAYS_LAST_DUE_1ST_VERSION']\ 
.agg({"returns": [np.min, np.max,np.mean]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_day_lastdue'] = tmp_merge['des1']
df['max_day_lastdue'] = tmp_merge['des2']
df['mean_day_lastdue'] = tmp_merge['des3']<merge>
solution.to_csv("ridge_sol.csv", index = False )