Kaggle competition: House Prices - Advanced Regression Techniques
tmp = previous[~previous['DAYS_LAST_DUE_1ST_VERSION'].isnull() ].sort_values(by=['SK_ID_CURR','DAYS_LAST_DUE_1ST_VERSION'])
 tmp = tmp.groupby(['SK_ID_CURR'] ).nth(-2 ).reset_index()
tmp = tmp[['SK_ID_CURR','DAYS_LAST_DUE_1ST_VERSION']]
tmp.columns = ['SK_ID_CURR','des']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['2nd_day_lastdue'] = tmp_merge['des']<merge>
%matplotlib inline
color = sns.color_palette()
sns.set_style('darkgrid')
 def ignore_warn(*args, **kwargs):
pass
warnings.warn = ignore_warn
pd.set_option('display.float_format', lambda x: '{:.3f}'.format(x))
print(check_output(["ls", ".. /input"] ).decode("utf8"))
warnings.filterwarnings('ignore')
tmp = previous.groupby(['SK_ID_CURR'])['sooner']\ 
.agg({"returns": [np.min, np.max,np.mean]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_sooner'] = tmp_merge['des1']
df['max_sooner'] = tmp_merge['des2']
df['mean_sooner'] = tmp_merge['des3']<merge>
from numpy import sqrt
tmp = previous.groupby(['SK_ID_CURR'])['SELLERPLACE_AREA']\ 
.agg({"returns": [np.min, np.max,np.mean]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_seller'] = tmp_merge['des1']
df['max_seller'] = tmp_merge['des2']
df['mean_seller'] = tmp_merge['des3']<merge>
train_data = pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/train.csv")
 test_data = pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/test.csv" )
for i in ['middle','low_normal','high','low_action']:
tmp = previous[(previous['NAME_CONTRACT_TYPE'] == 'Cash loans')&(previous['NAME_YIELD_GROUP'] == i)]
for df in [train,test]:
tmp1 = tmp.groupby(['SK_ID_CURR'])['SK_ID_PREV'].count().reset_index()
tmp1.columns = ['SK_ID_CURR','des']
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp1, on=['SK_ID_CURR'], how='left')
 df['count_' + str(i)] = tmp_merge['des'].fillna(0 )<feature_engineering>
idsUnique = len(set(train_data.Id))
idsTotal = train_data.shape[0]
idsDupli = idsTotal - idsUnique
print("Train data: there are " + str(idsDupli)+ " duplicate IDs for " + str(idsTotal)+ " total entries")
 train_data.drop("Id", axis = 1, inplace = True )
for df in [train,test]:
df['tmp'] = df[['count_middle','count_low_normal','count_high','count_low_action']].sum(axis=1)
 for i in ['middle','low_normal','high','low_action']:
df['ratio_' + i] = df['count_' + i]/df['tmp'] 
<merge>
idsUnique = len(set(test_data.Id))
idsTotal = test_data.shape[0]
idsDupli = idsTotal - idsUnique
print("Test data: there are " + str(idsDupli)+ " duplicate IDs for " + str(idsTotal)+ " total entries")
 test_data.drop("Id", axis = 1, inplace = True )
for i in ['middle','low_normal','high','low_action']:
tmp = previous[(previous['NAME_CONTRACT_TYPE'] == 'Consumer loans')&(previous['NAME_YIELD_GROUP'] == i)]
for df in [train,test]:
tmp1 = tmp.groupby(['SK_ID_CURR'])['SK_ID_PREV'].count().reset_index()
tmp1.columns = ['SK_ID_CURR','des']
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp1, on=['SK_ID_CURR'], how='left')
 df['count_' + str(i)+ '_v1'] = tmp_merge['des'].fillna(0 )<feature_engineering>
quantitative = list(train_data.dtypes[train_data.dtypes != "object"].index)
 len(quantitative )
for df in [train,test]:
df['tmp'] = df[['count_middle_v1','count_low_normal_v1','count_high_v1','count_low_action_v1']].sum(axis=1)
 for i in ['middle','low_normal','high','low_action']:
df['ratio_' + i +"_v1"] = df['count_' + i + "_v1"]/df['tmp'] 
<groupby>
qualitative = list(train_data.dtypes[train_data.dtypes == "object"].index)
 len(qualitative )
previous['tmp'] =(previous['AMT_ANNUITY'] * previous['CNT_PAYMENT'])/previous['AMT_CREDIT']
tmp = previous[(previous['NAME_CONTRACT_TYPE'] == 'Cash loans')&(previous['NAME_CONTRACT_STATUS'] != 'Approved')].groupby(['SK_ID_CURR'])['tmp']\ 
.agg({"returns": [np.min, np.max,np.mean]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_interest'] = tmp_merge['des1']
df['max_interest'] = tmp_merge['des2']
df['mean_interest'] = tmp_merge['des3']
tmp = previous[(previous['NAME_CONTRACT_TYPE'] == 'Consumer loans')&(previous['NAME_CONTRACT_STATUS'] != 'Approved')].groupby(['SK_ID_CURR'])['tmp']\ 
.agg({"returns": [np.min, np.max,np.mean]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_interest_v1'] = tmp_merge['des1']
df['max_interest_v1'] = tmp_merge['des2']
df['mean_interest_v1'] = tmp_merge['des3']<merge>
def encode(frame_train, frame_test,feature):
ordering = pd.DataFrame()
ordering['val'] = frame_train[feature].unique()
ordering.index = ordering.val
ordering['spmean'] = frame_train[[feature, 'SalePrice']].groupby(feature ).mean() ['SalePrice']
ordering = ordering.sort_values('spmean')
 ordering['ordering'] = range(1, ordering.shape[0]+1)
 ordering = ordering['ordering'].to_dict()
for cat, o in ordering.items() :
frame_train.loc[frame_train[feature] == cat, feature+'_E'] = o
frame_test.loc[frame_test[feature] == cat, feature+'_E'] = o
qual_encoded = []
for q in qualitative:
encode(train_data, test_data,q)
 qual_encoded.append(q+'_E')
 print(qual_encoded )
tmp = previous.groupby(['SK_ID_CURR'])['DAYS_FIRST_DRAWING']\ 
.agg({"returns": [np.min, np.max,np.mean]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_firstdraw'] = tmp_merge['des1']
df['max_firstdraw'] = tmp_merge['des2']
df['mean_firstdraw'] = tmp_merge['des3']<groupby>
ordering = pd.DataFrame()
ordering['val'] = train_data['MSZoning'].unique()
ordering.index = ordering.val
ordering['spmean'] = train_data[['MSZoning', 'SalePrice']].groupby('MSZoning' ).mean() ['SalePrice']
ordering = ordering.sort_values('spmean')
 ordering['ordering'] = range(1, ordering.shape[0]+1)
 ordering = ordering['ordering'].to_dict()
previous['tmp'] = previous['DAYS_FIRST_DRAWING'] - previous['DAYS_DECISION']
tmp = previous.groupby(['SK_ID_CURR'])['tmp']\ 
.agg({"returns": [np.min, np.max,np.mean]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_firstdraw_decision'] = tmp_merge['des1']
df['max_firstdraw_decision'] = tmp_merge['des2']
df['mean_firstdraw_decision'] = tmp_merge['des3']<groupby>
q = list(qualitative )
previous['tmp'] = previous['DAYS_FIRST_DUE'] - previous['DAYS_FIRST_DRAWING']
tmp = previous.groupby(['SK_ID_CURR'])['tmp']\ 
.agg({"returns": [np.min, np.max,np.mean]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_firstdraw_firstdue'] = tmp_merge['des1']
df['max_firstdraw_firstdue'] = tmp_merge['des2']
df['mean_firstdraw_firstdue'] = tmp_merge['des3']<groupby>
train_data = train_data[train_data.GrLivArea < 4500]
previous['tmp'] = previous['DAYS_LAST_DUE'] - previous['DAYS_FIRST_DRAWING']
tmp = previous.groupby(['SK_ID_CURR'])['tmp']\ 
.agg({"returns": [np.min, np.max,np.mean]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_firstdraw_lastdue'] = tmp_merge['des1']
df['max_firstdraw_lastdue'] = tmp_merge['des2']
df['mean_firstdraw_lastdue'] = tmp_merge['des3']<merge>
ntrain = train_data.shape[0]
ntest = test_data.shape[0]
y_train = train_data.SalePrice.values
all_data = pd.concat(( train_data, test_data),ignore_index = True)
 all_data.drop(['SalePrice'], axis=1, inplace=True)
 print("all_data size is : {}".format(all_data.shape))
tmp = bureau[(bureau['CREDIT_ACTIVE'] == "Active")].groupby(['SK_ID_CURR'])['SK_ID_BUREAU'].count().reset_index()
tmp.columns = ['SK_ID_CURR','des']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['count_active_bureau'] = tmp_merge['des'].fillna(0)
 tmp = bureau[bureau['CREDIT_ACTIVE'] != "Active"].groupby(['SK_ID_CURR'])['SK_ID_BUREAU'].count().reset_index()
tmp.columns = ['SK_ID_CURR','des']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['count_closed_bureau'] = tmp_merge['des'].fillna(0 )<merge>
print(all_data.isnull().sum() )
tmp = bureau[(bureau['CREDIT_TYPE'] == "Consumer credit")].groupby(['SK_ID_CURR'])['SK_ID_BUREAU'].count().reset_index()
tmp.columns = ['SK_ID_CURR','des']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['count_active_bureau_v2'] = tmp_merge['des'].fillna(0)
 tmp = bureau[(bureau['CREDIT_ACTIVE'] != "Active")&(bureau['CREDIT_TYPE'] == "Credit card")].groupby(['SK_ID_CURR'])['SK_ID_BUREAU'].count().reset_index()
tmp.columns = ['SK_ID_CURR','des']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['count_closed_bureau_v2'] = tmp_merge['des'].fillna(0 )<merge>
qx = list(set(qualitative ).difference(['Neighborhood']))
all_data.drop(qx,axis = 1,inplace=True )
tmp = bureau[(~bureau['CREDIT_TYPE'].isin(["Credit card","Consumer credit"])) ].groupby(['SK_ID_CURR'])['SK_ID_BUREAU'].count().reset_index()
tmp.columns = ['SK_ID_CURR','des']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['count_active_bureau_v3'] = tmp_merge['des'].fillna(0)
 tmp = bureau[(bureau['CREDIT_ACTIVE'] != "Active")&(~bureau['CREDIT_TYPE'].isin(["Credit card","Consumer credit"])) ].groupby(['SK_ID_CURR'])['SK_ID_BUREAU'].count().reset_index()
tmp.columns = ['SK_ID_CURR','des']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['count_closed_bureau_v3'] = tmp_merge['des'].fillna(0 )<merge>
all_data_na = pd.DataFrame(( all_data.isnull().sum() / len(all_data)) * 100)
 all_data_na = all_data_na[all_data_na[0] > 0]
all_data_na.sort_values(ascending=False,by = [0],inplace = True)
 all_data_na.head(20 )
bureau['AMT_CREDIT_SUM'].replace(0, np.nan, inplace = True)
 tmp = bureau[(bureau['CREDIT_ACTIVE'] == "Active")&(bureau['CREDIT_TYPE'] == "Consumer credit")].groupby(['SK_ID_CURR'])['AMT_CREDIT_SUM'].agg({"returns": [np.min, np.max,np.mean, np.sum]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3', 'des4']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_active_credit_bureau'] = tmp_merge['des1']
df['max_active_credit_bureau'] = tmp_merge['des2']
df['mean_active_credit_bureau'] = tmp_merge['des3']
df['sum_active_credit_bureau'] = tmp_merge['des4']
tmp = bureau[(bureau['CREDIT_ACTIVE'] != "Active")&(bureau['CREDIT_TYPE'] == "Consumer credit")].groupby(['SK_ID_CURR'])['AMT_CREDIT_SUM'].agg({"returns": [np.min, np.max,np.mean, np.sum]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3', 'des4']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_closed_credit_bureau'] = tmp_merge['des1']
df['max_closed_credit_bureau'] = tmp_merge['des2']
df['mean_closed_credit_bureau'] = tmp_merge['des3']
df['sum_closed_credit_bureau'] = tmp_merge['des4']<merge>
all_data.drop(['PoolQC_E','MiscFeature_E','Alley_E'],axis = 1,inplace=True )
bureau['AMT_CREDIT_SUM'].replace(0, np.nan, inplace = True)
 tmp = bureau[(bureau['CREDIT_TYPE'] == "Credit card")].groupby(['SK_ID_CURR'])['AMT_CREDIT_SUM'].agg({"returns": [np.min, np.max,np.mean, np.sum]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3', 'des4']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_active_credit_bureau_v1'] = tmp_merge['des1']
df['max_active_credit_bureau_v1'] = tmp_merge['des2']
df['mean_active_credit_bureau_v1'] = tmp_merge['des3']
df['sum_active_credit_bureau_v1'] = tmp_merge['des4'] 
<merge>
fea = list(all_data_na[all_data_na[0]<20].index)
 for f in fea:
all_data[f] = all_data.groupby('Neighborhood')[f].apply(lambda x: x.fillna(x.median()))
bureau['AMT_CREDIT_SUM'].replace(0, np.nan, inplace = True)
 tmp = bureau[(bureau['CREDIT_TYPE'] == "Car loan")].groupby(['SK_ID_CURR'])['AMT_CREDIT_SUM'].agg({"returns": [np.min, np.max,np.mean, np.sum]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3', 'des4']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_active_credit_bureau_v2'] = tmp_merge['des1']
df['max_active_credit_bureau_v2'] = tmp_merge['des2']
df['mean_active_credit_bureau_v2'] = tmp_merge['des3']
df['sum_active_credit_bureau_v2'] = tmp_merge['des4'] 
<merge>
all_data.drop('Neighborhood',axis = 1,inplace=True )
bureau['AMT_CREDIT_SUM'].replace(0, np.nan, inplace = True)
 tmp = bureau[(~bureau['CREDIT_TYPE'].isin(["Credit card","Consumer credit","Car loan"])) ].groupby(['SK_ID_CURR'])['AMT_CREDIT_SUM'].agg({"returns": [np.min, np.max,np.mean, np.sum]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3', 'des4']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_active_credit_bureau_v3'] = tmp_merge['des1']
df['max_active_credit_bureau_v3'] = tmp_merge['des2']
df['mean_active_credit_bureau_v3'] = tmp_merge['des3']
df['sum_active_credit_bureau_v3'] = tmp_merge['des4'] 
<merge>
all_data_na1 = pd.DataFrame(( all_data.isnull().sum() / len(all_data)) * 100)
 all_data_na1 = all_data_na1[all_data_na1[0] > 0]
tmp = bureau.groupby(['SK_ID_CURR'])['AMT_CREDIT_SUM'].agg({"returns": [np.min, np.max,np.mean, np.sum]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3', 'des4']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_credit_bureau'] = tmp_merge['des1']
df['max_credit_bureau'] = tmp_merge['des2']
df['mean_credit_bureau'] = tmp_merge['des3']
df['sum_credit_bureau'] = tmp_merge['des4']<merge>
all_data['FireplaceQu_E'] = all_data.groupby('KitchenQual_E')['FireplaceQu_E'].apply(lambda x: x.fillna(x.median()))
all_data['Fence_E'] = all_data.groupby('KitchenQual_E')['Fence_E'].apply(lambda x: x.fillna(x.median()))
tmp = bureau[(bureau['CREDIT_TYPE'] == "Consumer credit")].groupby(['SK_ID_CURR'])['DAYS_CREDIT_ENDDATE'].agg({"returns": [np.min, np.max,np.mean, np.sum]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3', 'des4']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_endate_bureau'] = tmp_merge['des1']
df['max_endate_bureau'] = tmp_merge['des2']
df['mean_endate_bureau'] = tmp_merge['des3']
df['sum_endate_bureau'] = tmp_merge['des4']
tmp = bureau[(~bureau['DAYS_CREDIT_ENDDATE'].isnull())&(( bureau['CREDIT_TYPE'] == "Consumer credit")) ].sort_values(by=['SK_ID_CURR','DAYS_CREDIT_ENDDATE'])
 tmp = tmp.groupby(['SK_ID_CURR'] ).nth(-2 ).reset_index()
tmp = tmp[['SK_ID_CURR','DAYS_CREDIT_ENDDATE']]
tmp.columns = ['SK_ID_CURR','des']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['1st_endate_bureau'] = tmp_merge['des']<merge>
numeric_dtypes = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
numerics2 = []
for i in all_data.columns:
if all_data[i].dtype in numeric_dtypes:
numerics2.append(i )
tmp = bureau[(bureau['CREDIT_TYPE'] == "Car loan")].groupby(['SK_ID_CURR'])['DAYS_CREDIT_ENDDATE'].agg({"returns": [np.min, np.max,np.mean, np.sum]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3', 'des4']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_endate_bureau_v1'] = tmp_merge['des1']
df['max_endate_bureau_v1'] = tmp_merge['des2']
df['mean_endate_bureau_v1'] = tmp_merge['des3']
df['sum_endate_bureau_v1'] = tmp_merge['des4']
tmp = bureau[(~bureau['DAYS_CREDIT_ENDDATE'].isnull())&(( bureau['CREDIT_TYPE'] == "Car loan")) ].sort_values(by=['SK_ID_CURR','DAYS_CREDIT_ENDDATE'])
 tmp = tmp.groupby(['SK_ID_CURR'] ).nth(-2 ).reset_index()
tmp = tmp[['SK_ID_CURR','DAYS_CREDIT_ENDDATE']]
tmp.columns = ['SK_ID_CURR','des']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['1st_endate_bureau_v1'] = tmp_merge['des']<merge>
num = list(set(quantitative ).difference(['YearBuilt','YearRemodAdd', 'MoSold', 'YrSold']))
tmp = bureau[(bureau['CREDIT_TYPE'] == "Consumer credit")].groupby(['SK_ID_CURR'])['DAYS_CREDIT'].agg({"returns": [np.min, np.max,np.mean, np.sum]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3', 'des4']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_startdate_bureau'] = tmp_merge['des1']
df['max_startdate_bureau'] = tmp_merge['des2']
df['mean_startdate_bureau'] = tmp_merge['des3']
df['sum_startdate_bureau'] = tmp_merge['des4']<merge>
skew_features = all_data.apply(lambda x: skew(x)).sort_values(ascending=False)
 high_skew = skew_features[abs(skew_features)> 0.75]
print("There are {} skewed numerical features to Box Cox transform".format(high_skew.shape[0]))
skew_index = high_skew.index
for i in skew_index:
all_data[i] = boxcox1p(all_data[i], boxcox_normmax(all_data[i] + 1))
tmp = bureau[(~bureau['DAYS_CREDIT_ENDDATE'].isnull())&(( bureau['CREDIT_TYPE'] == "Consumer credit")) ].sort_values(by=['SK_ID_CURR','DAYS_CREDIT_ENDDATE'])
 tmp = tmp.groupby(['SK_ID_CURR'] ).nth(-2 ).reset_index()
tmp = tmp[['SK_ID_CURR','DAYS_ENDDATE_FACT']]
tmp.columns = ['SK_ID_CURR','des']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['1st_endatefact_bureau'] = tmp_merge['des']<merge>
train = all_data[:ntrain]
test = all_data[ntrain:]
tmp = bureau[(bureau['CREDIT_TYPE'] == "Consumer credit")].groupby(['SK_ID_CURR'])['DAYS_ENDDATE_FACT'].agg({"returns": [np.min, np.max,np.mean, np.sum]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3', 'des4']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_endatefact_bureau'] = tmp_merge['des1']
df['max_endatefact_bureau'] = tmp_merge['des2']
df['mean_endatefact_bureau'] = tmp_merge['des3']
df['sum_endatefact_bureau'] = tmp_merge['des4']<merge>
kfolds = KFold(n_splits=10, shuffle=True, random_state=42)
 def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))
def cv_rmse(model, X=train):
rmse = np.sqrt(-cross_val_score(model, X, y_train, scoring="neg_mean_squared_error", cv=kfolds))
return(rmse )
bureau['tmp'] = bureau['DAYS_CREDIT_ENDDATE'] - bureau['DAYS_ENDDATE_FACT']
tmp = bureau.groupby(['SK_ID_CURR'])['tmp'].agg({"returns": [np.min, np.max,np.mean, np.sum]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3', 'des4']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_deltaendate_bureau'] = tmp_merge['des1']
df['max_deltaendate_bureau'] = tmp_merge['des2']
df['mean_deltaendate_bureau'] = tmp_merge['des3']
df['sum_deltaendate_bureau'] = tmp_merge['des4']<merge>
alphas_alt = [14.5, 14.6, 14.7, 14.8, 14.9, 15, 15.1, 15.2, 15.3, 15.4, 15.5]
alphas2 = [5e-05, 0.0001, 0.0002, 0.0003, 0.0004, 0.0005, 0.0006, 0.0007, 0.0008]
e_alphas = [0.0001, 0.0002, 0.0003, 0.0004, 0.0005, 0.0006, 0.0007]
e_l1ratio = [0.8, 0.85, 0.9, 0.95, 0.99, 1]
bureau['tmp'] =(bureau['DAYS_CREDIT_ENDDATE'] - bureau['DAYS_CREDIT'])
 tmp = bureau[(bureau['CREDIT_TYPE'] == "Consumer credit")].groupby(['SK_ID_CURR'])['tmp'].agg({"returns": [np.min, np.max,np.mean, np.sum]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3', 'des4']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_duration_bureau'] = tmp_merge['des1']
df['max_duration_bureau'] = tmp_merge['des2']
df['mean_duration_bureau'] = tmp_merge['des3']
df['sum_duration_bureau'] = tmp_merge['des4']
bureau['tmp'] =(bureau['DAYS_CREDIT_ENDDATE'] - bureau['DAYS_CREDIT'])
 tmp = bureau[(bureau['CREDIT_TYPE'] != "Consumer credit")].groupby(['SK_ID_CURR'])['tmp'].agg({"returns": [np.min, np.max,np.mean, np.sum]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3', 'des4']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_duration_bureau_v1'] = tmp_merge['des1']
df['max_duration_bureau_v1'] = tmp_merge['des2']
df['mean_duration_bureau_v1'] = tmp_merge['des3']
df['sum_duration_bureau_v1'] = tmp_merge['des4']<merge>
ridge = make_pipeline(RobustScaler() , RidgeCV(alphas=alphas_alt, cv=kfolds))
lasso = make_pipeline(RobustScaler() , LassoCV(max_iter=1e7, alphas=alphas2, random_state=42, cv=kfolds))
elasticnet = make_pipeline(RobustScaler() , ElasticNetCV(max_iter=1e7, alphas=e_alphas, cv=kfolds, l1_ratio=e_l1ratio))
svr = make_pipeline(RobustScaler() , SVR(C= 20, epsilon= 0.008, gamma=0.0003,))
bureau['tmp'] =(bureau['DAYS_ENDDATE_FACT'] - bureau['DAYS_CREDIT'])
 tmp = bureau[(bureau['CREDIT_TYPE'] == "Consumer credit")].groupby(['SK_ID_CURR'])['tmp'].agg({"returns": [np.min, np.max,np.mean, np.sum]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3', 'des4']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_durationfact_bureau'] = tmp_merge['des1']
df['max_durationfact_bureau'] = tmp_merge['des2']
df['mean_durationfact_bureau'] = tmp_merge['des3']
df['sum_durationfact_bureau'] = tmp_merge['des4']<groupby>
lightgbm = LGBMRegressor(objective='regression',
num_leaves=4,
learning_rate=0.01,
n_estimators=5000,
max_bin=200,
bagging_fraction=0.75,
bagging_freq=5,
bagging_seed=7,
feature_fraction=0.2,
feature_fraction_seed=7,
verbose=-1,
)
bureau['tmp'] =(bureau['DAYS_ENDDATE_FACT'] - bureau['DAYS_CREDIT_ENDDATE'])/(bureau['DAYS_CREDIT'] - bureau['DAYS_CREDIT_ENDDATE'])
 tmp = bureau[(bureau['CREDIT_TYPE'] == "Consumer credit")].groupby(['SK_ID_CURR'])['tmp'].agg({"returns": [np.min, np.max,np.mean, np.sum]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3', 'des4']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_sooner_bureau'] = tmp_merge['des1']
df['max_sooner_bureau'] = tmp_merge['des2']
df['mean_sooner_bureau'] = tmp_merge['des3']
df['sum_sooner_bureau'] = tmp_merge['des4']<groupby>
gbr = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05, max_depth=4, max_features='sqrt', min_samples_leaf=15, min_samples_split=10, loss='huber', random_state =42 )
bureau['tmp'] =(bureau['DAYS_ENDDATE_FACT'] - bureau['DAYS_CREDIT_ENDDATE'])/(bureau['DAYS_CREDIT'] - bureau['DAYS_CREDIT_ENDDATE'])
 tmp = bureau[(~bureau['CREDIT_TYPE'].isin(['Credit card','Consumer credit'])) ].groupby(['SK_ID_CURR'])['tmp'].agg({"returns": [np.min, np.max,np.mean, np.sum]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3', 'des4']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_sooner_bureau_v1'] = tmp_merge['des1']
df['max_sooner_bureau_v1'] = tmp_merge['des2']
df['mean_sooner_bureau_v1'] = tmp_merge['des3']
df['sum_sooner_bureau_v1'] = tmp_merge['des4']<merge>
xgboost = XGBRegressor(learning_rate=0.01,n_estimators=3460,
max_depth=3, min_child_weight=0,
gamma=0, subsample=0.7,
colsample_bytree=0.7,
objective='reg:linear', nthread=-1,
scale_pos_weight=1, seed=27,
reg_alpha=0.00006 )
bureau['tmp'] =(bureau['AMT_CREDIT_SUM'])/(bureau['DAYS_CREDIT'] - bureau['DAYS_CREDIT_ENDDATE'])
 tmp = bureau[bureau['CREDIT_TYPE'] == "Credit card"].groupby(['SK_ID_CURR'])['tmp'].agg({"returns": [np.min, np.max,np.mean, np.sum]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3', 'des4']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_annuity_bureau'] = tmp_merge['des1']
df['max_annuity_bureau'] = tmp_merge['des2']
df['mean_annuity_bureau'] = tmp_merge['des3']
df['sum_annuity_bureau'] = tmp_merge['des4']<groupby>
stack_gen = StackingCVRegressor(regressors=(ridge, lasso, elasticnet, gbr, xgboost, lightgbm),
meta_regressor=xgboost,
use_features_in_secondary=True )
bureau['tmp'] =(bureau['AMT_CREDIT_SUM'])/(bureau['DAYS_CREDIT'] - bureau['DAYS_CREDIT_ENDDATE'])
 tmp = bureau[(bureau['CREDIT_ACTIVE'] == "Active")&(bureau['CREDIT_TYPE'] == "Consumer credit")].groupby(['SK_ID_CURR'])['tmp'].agg({"returns": [np.min, np.max,np.mean, np.sum]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3', 'des4']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_annuity_bureau_v1'] = tmp_merge['des1']
df['max_annuity_bureau_v1'] = tmp_merge['des2']
df['mean_annuity_bureau_v1'] = tmp_merge['des3']
df['sum_annuity_bureau_v1'] = tmp_merge['des4']
tmp = bureau[(bureau['CREDIT_ACTIVE'] == "Active")&(bureau['CREDIT_TYPE'] == "Consumer credit")].groupby(['SK_ID_CURR'])['tmp'].agg({"returns": [np.min, np.max,np.mean, np.sum]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3', 'des4']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_annuity_bureau_v2'] = tmp_merge['des1']
df['max_annuity_bureau_v2'] = tmp_merge['des2']
df['mean_annuity_bureau_v2'] = tmp_merge['des3']
df['sum_annuity_bureau_v2'] = tmp_merge['des4']<merge>
score = cv_rmse(ridge)
 print("RIDGE: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() ,)
 score = cv_rmse(lasso)
 print("LASSO: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() ,)
 score = cv_rmse(elasticnet)
 print("elastic net: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() ,)
 score = cv_rmse(svr)
 print("SVR: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() ,)
 score = cv_rmse(lightgbm)
 print("lightgbm: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() ,)
 score = cv_rmse(gbr)
 print("gbr: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() ,)
 score = cv_rmse(xgboost)
 print("xgboost: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() , )
bureau['tmp'] =(bureau['AMT_CREDIT_SUM'])/(bureau['DAYS_CREDIT'] - bureau['DAYS_CREDIT_ENDDATE'])
 tmp = bureau[~bureau['CREDIT_TYPE'].isin(["Credit card","Consumer credit"])].groupby(['SK_ID_CURR'])['tmp'].agg({"returns": [np.min, np.max,np.mean, np.sum]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3', 'des4']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_annuity_bureau_v3'] = tmp_merge['des1']
df['max_annuity_bureau_v3'] = tmp_merge['des2']
df['mean_annuity_bureau_v3'] = tmp_merge['des3']
df['sum_annuity_bureau_v3'] = tmp_merge['des4']<categorify>
print('START Fit')
 print('stack_gen')
 stack_gen_model = stack_gen.fit(np.array(train), np.array(y_train))
print('elasticnet')
 elastic_model_full_data = elasticnet.fit(train, y_train)
 print('Lasso')
 lasso_model_full_data = lasso.fit(train, y_train)
 print('Ridge')
 ridge_model_full_data = ridge.fit(train, y_train)
 print('Svr')
 svr_model_full_data = svr.fit(train, y_train)
 print('GradientBoosting')
 gbr_model_full_data = gbr.fit(train, y_train)
 print('xgboost')
 xgb_model_full_data = xgboost.fit(train, y_train)
 print('lightgbm')
 lgb_model_full_data = lightgbm.fit(train, y_train )
bureau['AMT_CREDIT_SUM_DEBT_v1'] = bureau['AMT_CREDIT_SUM_DEBT'].replace(0, np.nan)
 tmp = bureau[(bureau['CREDIT_ACTIVE'] == "Active")&(bureau['CREDIT_TYPE'].isin(["Credit card"])) ].groupby(['SK_ID_CURR'])['AMT_CREDIT_SUM_DEBT_v1'].agg({"returns": [np.min, np.max,np.mean, np.sum]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3', 'des4']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_debt_bureau'] = tmp_merge['des1']
df['max_debt_bureau'] = tmp_merge['des2']
df['mean_debt_bureau'] = tmp_merge['des3']
df['sum_debt_bureau'] = tmp_merge['des4']
tmp = bureau[(bureau['CREDIT_ACTIVE'] == "Active")&(bureau['CREDIT_TYPE'].isin(["Consumer credit"])) ].groupby(['SK_ID_CURR'])['AMT_CREDIT_SUM_DEBT_v1'].agg({"returns": [np.min, np.max,np.mean, np.sum]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3', 'des4']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_debt_bureau_v1'] = tmp_merge['des1']
df['max_debt_bureau_v1'] = tmp_merge['des2']
df['mean_debt_bureau_v1'] = tmp_merge['des3']
df['sum_debt_bureau_v1'] = tmp_merge['des4']<groupby>
def blend_models_predict(X):
return(( 0.1 * elastic_model_full_data.predict(X)) + \ 
(0.1 * lasso_model_full_data.predict(X)) + \ 
(0.1 * ridge_model_full_data.predict(X)) + \ 
(0.1 * svr_model_full_data.predict(X)) + \ 
(0.1 * gbr_model_full_data.predict(X)) + \ 
(0.1 * xgb_model_full_data.predict(X)) + \ 
(0.1 * lgb_model_full_data.predict(X)) + \ 
(0.3 * stack_gen_model.predict(np.array(X))))
bureau['AMT_CREDIT_SUM_LIMIT_v1'] = bureau['AMT_CREDIT_SUM_LIMIT'].replace(0, np.nan)
 tmp = bureau[(bureau['CREDIT_TYPE'].isin(["Credit card"])) ].groupby(['SK_ID_CURR'])['AMT_CREDIT_SUM_LIMIT_v1'].agg({"returns": [np.min, np.max,np.mean, np.sum]})\ 
.reset_index()
tmp.columns = ['SK_ID_CURR','des1','des2','des3', 'des4']
for df in [train,test]:
tmp_merge = df[['SK_ID_CURR']]
tmp_merge = tmp_merge.merge(tmp, on=['SK_ID_CURR'], how='left')
 df['min_limit_bureau'] = tmp_merge['des1']
df['max_limit_bureau'] = tmp_merge['des2']
df['mean_limit_bureau'] = tmp_merge['des3']
df['sum_limit_bureau'] = tmp_merge['des4'] 
<categorify>
print('RMSLE score on train data:')
 print(rmsle(y_train, np.expm1(blend_models_predict(train)) )
<merge><EOS> 
test_data = pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/test.csv")
 output = pd.DataFrame({'Id': test_data.Id, 'SalePrice': np.floor(np.expm1(blend_models_predict(test)))})
 output.to_csv('my_submission.csv', index=False)
 print("Your submission was successfully saved!" )