Kaggle competition: House Prices - Advanced Regression Techniques
clfs = []
for i in range(10):
clf = lgb.LGBMClassifier(max_depth=-1, learning_rate=0.1, objective='multiclass',
random_state=314+i, silent=True, metric='None',
n_jobs=4, n_estimators=5000, class_weight='balanced')
 clf.set_params(**opt_parameters)
 clfs.append(( 'lgbm{}'.format(i), clf))
vc = VotingClassifierLGBM(clfs, voting='soft')
 del clfs
_ = vc.fit(X_tr.drop(lgb_drop_cols, axis=1), y_tr-1, **fit_params)
 clf_final = vc.estimators_[0]<compute_train_metric>
%matplotlib inline
global_score = f1_score(y_cv-1, clf_final.predict(X_cv.drop(lgb_drop_cols, axis=1)) , average='macro')
 vc.voting = 'soft'
global_score_soft = f1_score(y_cv-1, vc.predict(X_cv.drop(lgb_drop_cols, axis=1)) , average='macro')
 print('Validation score of a single LGBM Classifier: {:.4f}'.format(global_score))
print('Validation score of a VotingClassifier on 3 LGBMs with soft voting strategy: {:.4f}'.format(global_score_soft))<predict_on_test>
df_train=pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 df_test=pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv' )
lgb_cv_probs2 = clf_final.predict_proba(X_cv.drop(lgb_drop_cols, axis=1))
lgb_probs2 = clf_final.predict_proba(test.drop(lgb_drop_cols, axis=1))
lgb_preds2 = np.argmax(lgb_probs2, axis=1)+ 1
submission['Target_lgb3'] = lgb_preds2<define_variables>
df=pd.concat([df_train,df_test] ).reset_index(drop=True)
 df
cv_probs = cv_probs + lgb_cv_probs2 + lgb_cv_probs2
test_probs = test_probs + lgb_probs2 + lgb_probs2
model_count += 3<save_to_csv>
total=(df.isnull().sum() ).sort_values(ascending=False)
 percentage=(( df.isnull().sum() /df.isnull().count())*100 ).sort_values(ascending=False)
 missing_values=pd.concat([total,percentage],keys=['total','percentage'],axis=1)
 missing_values.head(40 )
submission[["Id", "Target_lgb3"]].to_csv("20180725_lgb3_preds.csv", header=["Id", "Target"], index=False )<define_variables>
df['PoolQC']=df['PoolQC'].fillna('None' )
cv_probs = cv_probs / model_count
test_probs = test_probs / model_count<prepare_output>
df['MiscFeature']=df['MiscFeature'].fillna('None' )
cv_predictions = np.argmax(cv_probs, axis=1)+ 1
test_predictions = np.argmax(test_probs, axis=1)+ 1<compute_test_metric>
df['Alley']=df['Alley'].fillna('None' )
print("Accuracy:", accuracy_score(y_cv, cv_predictions))
print("F1:", f1_score(y_cv, cv_predictions, average="micro"))<save_to_csv>
df['Fence']=df['Fence'].fillna('None' )
submission['Avg_probs'] = test_predictions
submission[["Id", "Avg_probs"]].to_csv("20180725_avg_preds.csv", header=["Id", "Target"], index=False )<import_modules>
df['FireplaceQu']=df['FireplaceQu'].fillna('None' )
%matplotlib inline
print(os.listdir(".. /input")) 
<define_variables>
df['LotFrontage']=df['LotFrontage'].fillna(df['LotFrontage'].median() )
def dprint(*args, **kwargs):
print("[{}] ".format(datetime.datetime.now().strftime("%Y-%m-%d %H:%M")) + \
" ".join(map(str,args)) , **kwargs)
 id_name = 'Id'
target_name = 'Target' 
<load_from_csv>
df['GarageArea']=df['GarageArea'].fillna(0)
 df['GarageArea'].isnull().sum()
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
<concatenate>
df['GarageQual'].value_counts()
train['is_test'] = 0
test['is_test'] = 1
df_all = pd.concat([train, test], axis=0)
<data_type_conversions>
def impute_GarageQual(cols):
GarageArea=cols[1]
GarageQual=cols[0]
if pd.isnull(GarageQual):
if 0<GarageArea<400:
return 'Fa'
elif 400<GarageArea<550:
return 'Ta'
elif 550<GarageArea<700:
return 'Gd'
else:
return 'Ex'
else:
return GarageQual
dprint('Clean features...')
 cols = ['dependency']
for c in tqdm(cols):
x = df_all[c].values
strs = []
for i, v in enumerate(x):
try:
val = float(v)
 except:
strs.append(v)
 val = np.nan
x[i] = val
strs = np.unique(strs)
 for s in strs:
df_all[c + '_' + s] = df_all[c].apply(lambda x: 1 if x == s else 0)
 df_all[c] = x
df_all[c] = df_all[c].astype(float)
 dprint("Done." )<feature_engineering>
df['GarageQual']=df['GarageQual'].fillna('None')
 df['GarageQual'].isnull().sum()
dprint("Extracting features...")
 def extract_features(df):
df['bedrooms_to_rooms'] = df['bedrooms']/df['rooms']
df['rent_to_rooms'] = df['v2a1']/df['rooms']
df['rent_to_bedrooms'] = df['v2a1']/df['bedrooms']
df['tamhog_to_rooms'] = df['tamhog']/df['rooms']
df['tamhog_to_bedrooms'] = df['tamhog']/df['bedrooms']
df['r4t3_to_tamhog'] = df['r4t3']/df['tamhog']
df['r4t3_to_rooms'] = df['r4t3']/df['rooms']
df['r4t3_to_bedrooms'] = df['r4t3']/df['bedrooms']
df['rent_to_r4t3'] = df['v2a1']/df['r4t3']
df['v2a1_to_r4t3'] = df['v2a1']/(df['r4t3'] - df['r4t1'])
 df['hhsize_to_rooms'] = df['hhsize']/df['rooms']
df['hhsize_to_bedrooms'] = df['hhsize']/df['bedrooms']
df['rent_to_hhsize'] = df['v2a1']/df['hhsize']
df['qmobilephone_to_r4t3'] = df['qmobilephone']/df['r4t3']
df['qmobilephone_to_v18q1'] = df['qmobilephone']/df['v18q1']
extract_features(train)
 extract_features(test)
 dprint("Done." )<data_type_conversions>
def impute_GarageYrBlt(cols):
GarageYrBlt=cols[0]
GarageQual=cols[1]
if pd.isnull(GarageYrBlt):
if GarageQual=='Po':
return 1920
elif GarageQual=='Fa':
return 1930
elif GarageQual=='Ta':
return 1975
elif GarageQual=='Gd':
return 1980
elif GarageQual=='Ex':
return 2000
else:
return 2010
else:
return GarageYrBlt
def encode_data(df):
yes_no_map = {'no': 0, 'yes': 1}
df['dependency'] = df['dependency'].replace(yes_no_map ).astype(np.float32)
 df['edjefe'] = df['edjefe'].replace(yes_no_map ).astype(np.float32)
 df['edjefa'] = df['edjefa'].replace(yes_no_map ).astype(np.float32)
 df['idhogar'] = LabelEncoder().fit_transform(df['idhogar'])
<categorify>
df['GarageYrBlt'].isnull().sum()
dprint("Encoding Data.... ")
 encode_data(train)
 encode_data(test)
 dprint("Done..." )<feature_engineering>
df['GarageYrBlt']=df['GarageYrBlt'].fillna(0)
 df['GarageYrBlt'].isnull().sum()
def do_features(df):
feats_div = [('children_fraction', 'r4t1', 'r4t3'), 
('working_man_fraction', 'r4h2', 'r4t3'), 
('all_man_fraction', 'r4h3', 'r4t3'), 
('human_density', 'tamviv', 'rooms'), 
('human_bed_density', 'tamviv', 'bedrooms'), 
('rent_per_person', 'v2a1', 'r4t3'), 
('rent_per_room', 'v2a1', 'rooms'), 
('mobile_density', 'qmobilephone', 'r4t3'), 
('tablet_density', 'v18q1', 'r4t3'), 
('mobile_adult_density', 'qmobilephone', 'r4t2'), 
('tablet_adult_density', 'v18q1', 'r4t2'),
]
feats_sub = [('people_not_living', 'tamhog', 'tamviv'), 
('people_weird_stat', 'tamhog', 'r4t3')]
for f_new, f1, f2 in feats_div:
df['fe_' + f_new] =(df[f1] / df[f2] ).astype(np.float32)
 for f_new, f1, f2 in feats_sub:
df['fe_' + f_new] =(df[f1] - df[f2] ).astype(np.float32)
 aggs_num = {'age': ['min', 'max', 'mean'],
'escolari': ['min', 'max', 'mean']
}
aggs_cat = {'dis': ['mean']}
for s_ in ['estadocivil', 'parentesco', 'instlevel']:
for f_ in [f_ for f_ in df.columns if f_.startswith(s_)]:
aggs_cat[f_] = ['mean', 'count']
for name_, df_ in [('18', df.query('age >= 18')) ]:
df_agg = df_.groupby('idhogar' ).agg({**aggs_num, **aggs_cat} ).astype(np.float32)
 df_agg.columns = pd.Index(['agg' + name_ + '_' + e[0] + "_" + e[1].upper() for e in df_agg.columns.tolist() ])
 df = df.join(df_agg, how='left', on='idhogar')
 del df_agg
df.drop([f_ for f_ in df.columns if f_.startswith('SQB')or f_ == 'agesq'], axis=1, inplace=True)
 df.drop(['Id', 'idhogar'], axis=1, inplace=True)
 df.drop(['hhsize', 'female', 'area2'], axis=1, inplace=True)
 return df 
<feature_engineering>
def impute_GarageCond(cols):
GarageArea=cols[1]
GarageCond=cols[0]
if pd.isnull(GarageCond):
if 0<GarageArea<400:
return 'Fa'
elif 400<GarageArea<550:
return 'Ta'
elif 550<GarageArea<700:
return 'Gd'
else:
return 'Ex'
else:
return GarageCond
dprint("Do_feature Engineering.... ")
 train = do_features(train)
 test = do_features(test)
 dprint("Done.... " )<categorify>
df['GarageCond']=df['GarageCond'].fillna('None')
 df['GarageCond'].isnull().sum()
dprint("Fill Na value.... ")
 train = train.fillna(0)
 test = test.fillna(0)
 dprint("Done.... " )<prepare_x_and_y>
df['GarageType'].value_counts()
cols_to_drop = [
id_name,
target_name,
]
X = train.drop(cols_to_drop, axis=1, errors='ignore')
 y = train[target_name].values 
<set_options>
def impute_GarageType(cols):
GarageArea=cols[1]
GarageType=cols[0]
if pd.isnull(GarageType):
if 0<GarageArea<400:
return 'Detchd'
elif 400<GarageArea<600:
return 'Attchd'
elif 600<GarageArea<700:
return 'BuiltIn'
else:
return '2Types'
else:
return GarageType
gc.collect()<choose_model_class>
df['GarageType']=df['GarageType'].fillna('None')
 df['GarageType'].isnull().sum()
def get_lgb_model() :
lgb_model = lgb.LGBMClassifier(max_depth=-1, learning_rate=0.2, objective='multiclass',
silent=True, metric='None',
n_jobs=4, n_estimators=5000, class_weight='balanced',
colsample_bytree= 0.93,
min_child_samples= 56, num_leaves= 19,
subsample= 0.84)
 return lgb_model<predict_on_test>
df['TotalBsmtSF']=df['TotalBsmtSF'].fillna(df['TotalBsmtSF'].mean())
 df['TotalBsmtSF'].isnull().sum()
dprint("Prepared Model.....")
 lgb_model = get_lgb_model()
lgb_model.fit(X, y)
 dprint("Done Model.....")
 dprint("Predict Test Value.....")
 target_lgb = lgb_model.predict(test)
 dprint("Done Prediction....." )<train_model>
def impute_BsmtExposure(cols):
TotalBsmtSF=cols[1]
BsmtExposure=cols[0]
if pd.isnull(BsmtExposure):
if 0<TotalBsmtSF<900:
return 'No'
elif 900<TotalBsmtSF<1000:
return 'Mn'
elif 1000<TotalBsmtSF<1400:
return 'Av'
else:
return 'Gd'
else:
return BsmtExposure
<load_from_csv>
df['BsmtExposure']=df['BsmtExposure'].fillna('None')
 df['BsmtExposure'].isnull().sum()
sub = pd.read_csv(".. /input/sample_submission.csv" )<prepare_output>
def impute_BsmtCond(cols):
TotalBsmtSF=cols[1]
BsmtCond=cols[0]
if pd.isnull(BsmtCond):
if 0<TotalBsmtSF<700:
return 'Fa'
elif 700<TotalBsmtSF<1000:
return 'TA'
else:
return 'Gd'
else:
return BsmtCond
sub['Target'] = target_lgb<save_to_csv>
df['BsmtCond']=df['BsmtCond'].fillna('None')
 df['BsmtCond'].isnull().sum()
sub.to_csv("cat_boost.csv", index= False )<set_options>
def impute_BsmtQual(cols):
TotalBsmtSF=cols[1]
BsmtQual=cols[0]
if pd.isnull(BsmtQual):
if 0<TotalBsmtSF<800:
return 'TA'
elif 800<TotalBsmtSF<1400:
return 'Gd'
else:
return 'Ex'
else:
return BsmtQual
warnings.filterwarnings('ignore')
 %matplotlib inline<create_dataframe>
df['BsmtQual']=df['BsmtQual'].fillna('None')
 df['BsmtQual'].isnull().sum()
pd.set_option('display.max_columns', None )<load_from_csv>
df['BsmtFinType2']=df['BsmtFinType2'].fillna('NA')
 df['BsmtFinType2'].isnull().sum()
df_train = pd.read_csv('.. /input/train.csv')
 df_test = pd.read_csv('.. /input/test.csv')
 train_id = df_train['Id']
test_id = df_test['Id']
train_idhogar = df_train['idhogar']
test_idhogar = df_train['idhogar']
df_train.drop(columns=['Id'], inplace=True)
 df_test.drop(columns=['Id'], inplace=True)
 print("Shape of train data: ", df_train.shape)
 print("Shape of test data: ", df_test.shape)
 ntrain = df_train.shape[0]
ntest = df_test.shape[0]
all_data = pd.concat(( df_train, df_test)).reset_index(drop=True )<predict_on_test>
def impute_BsmtFinType1(cols):
TotalBsmtSF=cols[1]
BsmtFinType1=cols[0]
if pd.isnull(BsmtFinType1):
if 0<TotalBsmtSF<800:
return 'Unf'
else:
return 'GLQ'
else:
return BsmtFinType1
print("The feature that we need to predict: ", set(df_train.columns)- set(df_test.columns))<groupby>
df['BsmtFinType1']=df['BsmtFinType1'].fillna('None')
 df['BsmtFinType1'].isnull().sum()
outside_wall_material_df = df_train_heads.groupby('Target' ).sum() [['paredblolad', 'paredzocalo', 'paredpreb', 'pareddes', 'paredmad',
'paredzinc', 'paredfibras', 'paredother']]
outside_wall_material_df<groupby>
df['MasVnrArea']=df['MasVnrArea'].fillna(df['MasVnrArea'].mean() )
floor_material_df = df_train_heads.groupby('Target' ).sum() [['pisomoscer', 'pisocemento', 'pisoother',
'pisonatur', 'pisonotiene', 'pisomadera']]
floor_material_df<groupby>
df['MSZoning']=df['MSZoning'].fillna('RL' )
toilet_df = df_train_heads.groupby('Target' ).sum() [['sanitario1', 'sanitario2', 'sanitario3', 'sanitario5',
'sanitario6']]
toilet_df<groupby>
df['Utilities']=df['Utilities'].fillna('AllPub' )
rubbish_disposal_df = df_train_heads.groupby('Target' ).sum() [['elimbasu1', 'elimbasu2', 'elimbasu3',
'elimbasu4', 'elimbasu5', 'elimbasu6']]
rubbish_disposal_df<groupby>
df['Functional']=df['Functional'].fillna('Typ' )
roof_material_df = df_train_heads.groupby('Target' ).sum() [['techozinc', 'techoentrepiso', 'techocane', 'techootro']]
roof_material_df<groupby>
df['BsmtFullBath']=df['BsmtFullBath'].fillna(df['BsmtFullBath'].mean() )
water_provision_df = df_train_heads.groupby('Target' ).sum() [['abastaguadentro', 'abastaguafuera', 'abastaguano']]
water_provision_df<groupby>
df['BsmtHalfBath']=df['BsmtHalfBath'].fillna(df['BsmtHalfBath'].mean() )
electricity_df = df_train_heads.groupby('Target' ).sum() [['public', 'planpri', 'noelec', 'coopele']]
electricity_df<groupby>
df['GarageArea']=df['GarageArea'].fillna(df['GarageArea'].mean() )
cooking_energy_df = df_train_heads.groupby('Target' ).sum() [['energcocinar1', 'energcocinar2', 'energcocinar3',
'energcocinar4']]
cooking_energy_df<groupby>
df['BsmtFinSF2']=df['BsmtFinSF2'].fillna(df['BsmtFinSF2'].mean() )
avg_household_size_df = df_train_heads.groupby('Target' ).mean() ['hhsize']
avg_household_size_df<feature_engineering>
df['Exterior1st']=df['Exterior1st'].fillna(method='ffill' )
urban_rural_df = df_train_heads.groupby('Target' ).sum() [['area1', 'area2']]
urban_rural_df['UrbanPercentage'] = urban_rural_df['area1'] * round(( 100/sum(urban_rural_df['area1'])) , 6)
 urban_rural_df['RuralPercentage'] = urban_rural_df['area2'] * round(( 100/sum(urban_rural_df['area2'])) , 6)
 urban_rural_df<groupby>
df['TotalBsmtSF']=df['TotalBsmtSF'].fillna(method='ffill' )
region_df = df_train_heads.groupby('Target' ).sum() [['lugar1', 'lugar2', 'lugar3', 'lugar4', 'lugar5', 'lugar6']]
region_df<count_values>
df['GarageCars']=df['GarageCars'].fillna(0 )
round(((all_data.shape[0] - sum(all_data['v2a1'].value_counts())) / all_data.shape[0])* 100, 2 )<drop_column>
df['BsmtUnfSF']=df['BsmtUnfSF'].fillna(method='ffill' )
all_data.drop(columns=['sanitario1', 'sanitario6',
'elimbasu4', 'elimbasu5', 'elimbasu6',
'techozinc', 'techoentrepiso', 'techocane', 'techootro',
'abastaguadentro', 'abastaguafuera', 'abastaguano',
'public', 'planpri', 'noelec', 'coopele'], inplace=True )<data_type_conversions>
df['Electrical']=df['Electrical'].fillna(method='ffill' )
yes_no_map = {'no': 0, 'yes': 1}
all_data['dependency'] = all_data['dependency'].replace(yes_no_map ).astype(np.float32)
 all_data['edjefe'] = all_data['edjefe'].replace(yes_no_map ).astype(np.float32)
 all_data['edjefa'] = all_data['edjefa'].replace(yes_no_map ).astype(np.float32 )<count_unique_values>
df['BsmtFinSF1']=df['BsmtFinSF1'].fillna(df['BsmtFinSF1'].mean() )
num_binary_features = []
for feature in all_data.columns:
if sorted(df_train[feature].unique())in [[0, 1], [0], [1]]:
num_binary_features.append(feature)
 print("Total number of binary-numerical features: ", len(num_binary_features))
print("Binary-numerical features: ")
 num_binary_features<count_unique_values>
df['KitchenQual']=df['KitchenQual'].fillna(method='ffill' )
num_non_binary_features = [feature for feature in all_data.columns if feature not in num_binary_features]
print("Total number of non-binary-numerical features: ", len(num_non_binary_features))
print("Non-binary numerical features: ")
 num_non_binary_features_dict = {feature: len(all_data[feature].unique())for feature in num_non_binary_features}
num_non_binary_features_sorted = sorted(num_non_binary_features_dict,
key=lambda feature: num_non_binary_features_dict[feature],
reverse=True)
 num_non_binary_features_len_sorted = [num_non_binary_features_dict[feature] for feature in num_non_binary_features_sorted]
plt.figure(figsize=(16, 16))
barplot_with_anotate(num_non_binary_features_sorted, num_non_binary_features_len_sorted);
plt.ylabel("No.of unique values");
plt.xlabel("Non-binary numerical features");<compute_test_metric>
df['SaleType']=df['SaleType'].fillna(method='ffill' )
missing_features(all_data, all_data.columns )<data_type_conversions>
df['Exterior2nd']=df['Exterior2nd'].fillna(method='ffill' )
all_data.loc[:, 'v2a1'].fillna(0, inplace=True )<data_type_conversions>
df=df.drop(['Id'],axis=1 )
all_data.loc[:, 'v18q1'].fillna(0, inplace=True )<data_type_conversions>
df=pd.get_dummies(df )
all_data.loc[:, 'meaneduc'].fillna(all_data['meaneduc'].mean() , inplace=True)
 all_data.loc[:, 'SQBmeaned'].fillna(all_data['SQBmeaned'].mean() , inplace=True )<drop_column>
df.isnull().sum()
all_data.drop(columns=['rez_esc'], inplace=True )<feature_engineering>
train=df.iloc[:1460,:]
all_data['WallQual'] = all_data['epared1'] + 2*all_data['epared2'] + 3*all_data['epared3']
all_data['RoofQual'] = all_data['etecho1'] + 2*all_data['etecho2'] + 3*all_data['etecho3']
all_data['FloorQual'] = all_data['eviv1'] + 2*all_data['eviv2'] + 3*all_data['eviv3']
all_data['EducationLevel'] = all_data['instlevel1'] + 2*all_data['instlevel2'] + 3*all_data['instlevel3'] + \
4*all_data['instlevel4'] + 5*all_data['instlevel5'] + 6*all_data['instlevel6'] + 7*all_data['instlevel7'] + \
8*all_data['instlevel8'] + 9*all_data['instlevel9']<drop_column>
test=df.iloc[1460:,:]
all_data.drop(columns=['epared1', 'epared2', 'epared3',
'etecho1', 'etecho2', 'etecho3',
'eviv1', 'eviv2', 'eviv3',
'instlevel1', 'instlevel2', 'instlevel3', 'instlevel4', 'instlevel5',
'instlevel6', 'instlevel7', 'instlevel8', 'instlevel9'], inplace=True )<drop_column>
X_train=train.drop(["SalePrice"],axis=1)
 y_train=train['SalePrice']
redundant_features = ['r4t1', 'r4t2', 'r4t3', 'tamhog', 'tamviv', 'hhsize', 'r4t3', 'v18q', 'mobilephone']
all_data.drop(columns=redundant_features, inplace=True )<feature_engineering>
test=test.drop(['SalePrice'],axis=1 )
all_data['RentPerRoom'] = all_data['v2a1'] / all_data['rooms']
all_data['AdultsPerRoom'] = all_data['hogar_adul'] / all_data['rooms']
all_data['AdultsPerBedroom'] = all_data['hogar_adul'] / all_data['bedrooms']<define_variables>
XGB = XGBRegressor(colsample_bytree=0.4603, gamma=0.0468,
learning_rate=0.05, max_depth=3,
min_child_weight=1.7817, n_estimators=2200,
reg_alpha=0.4640, reg_lambda=0.8571,
subsample=0.5213, silent=1,
random_state =7, nthread = -1)
 XGB.fit(X_train,y_train )
ind_bool = ['dis', 'male', 'female', 'estadocivil1', 'estadocivil2', 'estadocivil3',
'estadocivil4', 'estadocivil5', 'estadocivil6', 'estadocivil7',
'parentesco1', 'parentesco2', 'parentesco3', 'parentesco4', 'parentesco5',
'parentesco6', 'parentesco7', 'parentesco8', 'parentesco9', 'parentesco10',
'parentesco11', 'parentesco12', 'EducationLevel']
ind_ordered = ['escolari', 'age']<groupby>
LGBM = LGBMRegressor(objective='regression',num_leaves=5,
learning_rate=0.05, n_estimators=720,
max_bin = 55, bagging_fraction = 0.8,
bagging_freq = 5, feature_fraction = 0.2319,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf =6, min_sum_hessian_in_leaf = 11)
 LGBM.fit(X_train,y_train )
f = lambda x: x.std(ddof=0)
 f.__name__ = 'std_0'
ind_agg = all_data.groupby('idhogar')[ind_ordered + ind_bool].agg(['mean', 'max', 'min', 'sum', f])
 new_cols = []
for col in ind_agg.columns.levels[0]:
for stat in ind_agg.columns.levels[1]:
new_cols.append(f'{col}-{stat}')
 ind_agg.columns = new_cols
ind_agg.head()<merge>
GBoost = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05,
max_depth=4, max_features='sqrt',
min_samples_leaf=15, min_samples_split=10,
loss='huber', random_state =5)
 GBoost.fit(X_train,y_train )
print("Original number of features:", all_data.shape[1])
 all_data = all_data.merge(ind_agg, on = 'idhogar', how = 'left')
 print("Number of features after merging transformed individual level features", all_data.shape[1])
 all_data.drop(columns=ind_bool+ind_ordered, inplace=True)
 print("Number of features after dropping the individual level features", all_data.shape[1] )<import_modules>
y_pred_XGB=XGB.predict(test)
 y_pred_LGBM=LGBM.predict(test)
 y_pred_GB=GBoost.predict(test )
from sklearn.metrics import f1_score, make_scorer
import lightgbm as lgb
import xgboost as xgb
from sklearn.model_selection import RandomizedSearchCV
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import StratifiedKFold<drop_column>
y_pred_XGB=pd.DataFrame(y_pred_XGB)
 y_pred_LGBM=pd.DataFrame(y_pred_LGBM)
 y_pred_GB=pd.DataFrame(y_pred_GB )
all_data.drop(columns=['idhogar'], inplace=True )<drop_column>
y_pred= 0.20 * y_pred_XGB[0] + 0.60 * y_pred_LGBM[0] + 0.20 * y_pred_GB[0]
y_pred
<prepare_x_and_y><EOS> 
sub=pd.concat([df_test['Id'],pd.DataFrame(y_pred)],axis=1)
 sub.columns=['Id','SalePrice']
sub.to_csv('submission.csv',index=False)
 sub.head()