Kaggle competition: House Prices - Advanced Regression Techniques
dprint("Do_feature Engineering.... ")
 train = do_features(train)
 test = do_features(test)
 dprint("Done.... " )<categorify>
%matplotlib inline
dprint("Fill Na value.... ")
 train = train.fillna(0)
 test = test.fillna(0)
 dprint("Done.... " )<prepare_x_and_y>
train_raw=pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 test_raw=pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv' )
cols_to_drop = [
id_name,
target_name,
]
X = train.drop(cols_to_drop, axis=1, errors='ignore')
 y = train[target_name].values<set_options>
train_raw = train_raw.drop(train_raw[(train_raw['GrLivArea']>4000)&(train_raw['SalePrice']<300000)].index )
gc.collect()<save_to_csv>
combine=pd.concat(( train_raw,test_raw)).reset_index(drop=True)
 combine.shape
%%time
svm_model = SVC(kernel='rbf', gamma=0.8, C=12)
 fit = svm_model.fit(X, y)
 pred = fit.predict(test)
 sub = pd.read_csv(".. /input/sample_submission.csv")
 sub["Target"] = pred
sub.to_csv("submission_svm.csv" ,index=False )<save_to_csv>
pd.set_option('display.max_rows', None)
 combine.isnull().sum().sum()
%%time
xgb_model = xgb.XGBClassifier(learning_rate= 0.1, n_estimators= 1000, max_depth= 5, min_child_weight= 1, gamma= 0,
subsample= 0.9, colsample_bytree= 0.8, objective= "multi:softmax", scale_pos_weight= 1,
eval_metric= "merror", silent= 1, verbose= False, num_class= 5, seed= 27)
 fit = xgb_model.fit(X, y)
 pred = fit.predict(test)
 sub = pd.read_csv(".. /input/sample_submission.csv")
 sub["Target"] = pred
sub.to_csv("submission_xgb.csv" ,index=False )<load_from_csv>
nullity=combine.isnull().sum() [ combine.isnull().sum() != 0]
nullity, nullity.shape
%%time
lgbm_model = lgb.LGBMClassifier(
 n_estimators=2000,
learning_rate=0.1,
num_leaves=123,
colsample_bytree=.8,
subsample=.7,
max_depth=15,
reg_alpha=.1,
reg_lambda=.1,
min_split_gain=.01,
min_child_weight=2,
scale_pos_weight=5, 
)
 fit = lgbm_model.fit(X, y)
 pred = fit.predict(test)
 sub = pd.read_csv(".. /input/sample_submission.csv")
 sub["Target"] = pred
sub.to_csv("submission_lgbm.csv" ,index=False )<save_to_csv>
combine.MSZoning.value_counts()
%%time
rf_model = RandomForestClassifier(
 n_jobs=4,
class_weight='balanced',
n_estimators=325,
max_depth=5 
)
 fit = lgbm_model.fit(X, y)
 pred = fit.predict(test)
 sub = pd.read_csv(".. /input/sample_submission.csv")
 sub["Target"] = pred
sub.to_csv("submission_rf.csv" ,index=False )<load_from_csv>
combine['MSZoning']=combine['MSZoning'].fillna('None')
 combine.MSZoning.value_counts()
%matplotlib inline
train = pd.read_csv('.. /input/train.csv')
 print('Shape of train set {}'.format(train.shape))
test = pd.read_csv('.. /input/test.csv')
 print('Shape of test set {}'.format(test.shape)) 
<categorify>
combine['LotFrontage']=combine.groupby('Neighborhood')['LotFrontage'].transform(lambda x:x.fillna(x.median()))
def onehot_encode(cat_feat, data):

 encoded = pd.get_dummies(data[cat_feat], prefix=cat_feat, dummy_na=True)
 res = pd.concat([data.drop(columns=[cat_feat]), encoded], axis='columns')
 return res
def add_quality(df, componente='pared', component='wall'):
for i in [1,2,3]:
i_quality =(df['e{}{}'.format(componente, i)] == 1)
 df.loc[i_quality, '{}_quality'.format(component)] = i
return df
def to_english(df, sp_pre='pared', eng_pre='wall_', translate=None):

 for sp in translate.keys() :
spanish_name = sp_pre + '{}'.format(sp)
 english_name = eng_pre + '{}'.format(translate[sp])
 df.rename(columns={spanish_name: english_name}, inplace=True)
 return df<count_unique_values>
combine.Alley.value_counts()
test['Target'] = np.nan
data_all = pd.concat([train, test])
 print('Shape of all data: {}'.format(data_all.shape))
n_house = data_all['idhogar'].nunique()
print('<drop_column>
combine['Alley']=combine['Alley'].fillna('None')
 combine.Alley.value_counts()
n_row = data_all.shape[0]
n_null = data_all.drop('Target', axis='columns' ).isnull().values.sum(axis=0)
 columns = list(data_all.columns)
 columns.remove('Target')
 pd.DataFrame({'column': columns,
'n_null': n_null} ).sort_values('n_null', ascending=False ).head(10 )<define_variables>
combine.Utilities.value_counts()
def mk_derived_feats(df):
feats_div = [('children_fraction', 'r4t1', 'r4t3'), 
('working_man_fraction', 'r4h2', 'r4t3'), 
('all_man_fraction', 'r4h3', 'r4t3'), 
('human_density', 'tamviv', 'rooms'), 
('human_bed_density', 'tamviv', 'bedrooms'), 
('mobile_density', 'qmobilephone', 'r4t3'), 
('mobile_adult_density', 'qmobilephone', 'r4t2'),
]
feats_sub = [('people_not_living', 'tamhog', 'tamviv'), 
('people_weird_stat', 'tamhog', 'r4t3')]
for f_new, f1, f2 in feats_div:
df['fe_' + f_new] =(df[f1] / df[f2] ).astype(np.float32)
 for f_new, f1, f2 in feats_sub:
df['fe_' + f_new] =(df[f1] - df[f2] ).astype(np.float32)
 return df
def mk_agg_feats(df):
aggs_num = {'age': ['min', 'max', 'mean'],
'escolari': ['min', 'max', 'mean']
}
aggs_cat = {'dis': ['sum', 'mean']}
for s_ in ['estadocivil', 'parentesco', 'instlevel']:
for f_ in [f_ for f_ in df.columns if f_.startswith(s_)]:
aggs_cat[f_] = ['mean', 'count']
for name_, df_ in [('18', df.query('age >= 18')) ]:
df_agg = df_.groupby('idhogar' ).agg({**aggs_num, **aggs_cat} ).astype(np.float32)
 df_agg.columns = pd.Index(['agg' + name_ + '_' + e[0] + "_" + e[1].upper() for e in df_agg.columns.tolist() ])
 df = df.join(df_agg, how='left', on='idhogar')
 del df_agg
return df
def drop_redundant(df):
df.drop([f_ for f_ in df.columns if f_.startswith('SQB')or f_ == 'agesq'], axis=1, inplace=True)
 df.drop(['hhsize', 'area2'], axis=1, inplace=True)
 return df<categorify>
test_raw.Utilities.value_counts()
yes_no_map = {'no': 0, 'yes': 1}
data_all['dependency'] = data_all['dependency'].replace(yes_no_map ).astype(np.float32)
 data_all['edjefe'] = data_all['edjefe'].replace(yes_no_map ).astype(np.float32)
 data_all['edjefa'] = data_all['edjefa'].replace(yes_no_map ).astype(np.float32)
 data_all['idhogar'] = LabelEncoder().fit_transform(data_all['idhogar'])
 data_all = mk_derived_feats(data_all)
 data_all = mk_agg_feats(data_all)
 data_all = drop_redundant(data_all )<define_variables>
combine=combine.drop(['Utilities'],axis=1)
 combine.Exterior1st.value_counts()
fe_feats = [ff for ff in data_all.columns if ff.startswith('fe_')]
agg_feats = [ff for ff in data_all.columns if ff.startswith('agg')]<define_variables>
combine['Exterior1st']=combine['Exterior1st'].fillna('Other')
 combine.Exterior1st.value_counts()
basic_feats = ['dependency'] 
<categorify>
combine['Exterior2nd']=combine['Exterior2nd'].fillna('Other')
 combine.Exterior2nd.value_counts()
head_df.loc[head_df['male'] == 1, 'head_gender'] = 'male'
head_df.loc[head_df['female'] == 1, 'head_gender'] = 'female'
print('Shape of head_df: {}'.format(head_df.shape))
head_df = onehot_encode('head_gender', head_df)
 head_gender_feats = [cc for cc in head_df.columns if 'head_gender' in cc]
head_gender_feats<feature_engineering>
combine.MasVnrType.value_counts()
for i in range(1, 10):
head_df.loc[head_df['instlevel{}'.format(i)] == 1, 'head_edu_level'] = i
head_df = head_df.rename(columns={'escolari': 'head_school_years'} )<merge>
combine['MasVnrType']=combine['MasVnrType'].fillna('None')
 combine.MasVnrType.value_counts()
cols = ['idhogar', 'head_school_years', 'head_edu_level'] + head_gender_feats
data_all = pd.merge(data_all, head_df[cols], how='left', on='idhogar')
 print(data_all.shape )<define_variables>
combine['MasVnrArea']=combine['MasVnrArea'].fillna(0)
 combine.MasVnrArea.isnull().sum()
house_head_feats = ['head_school_years', 'head_edu_level'] + head_gender_feats<categorify>
combine.BsmtQual.value_counts()
data_all = add_quality(data_all, componente='pared', component='wall')
 data_all = add_quality(data_all, componente='techo', component='roof')
 data_all = add_quality(data_all, componente='viv', component='floor')
 print(data_all.shape )<categorify>
combine['BsmtQual']=combine['BsmtQual'].fillna('None')
 combine.BsmtQual.value_counts()
translate = {'blolad': 'block',
'zocalo': 'socket',
'preb': 'cement',
'des': 'waste',
'mad': 'wood',
'zinc': 'zink',
'fibras': 'natural_fibers',
'other': 'other'}
data_all = to_english(data_all, sp_pre='pared', eng_pre='wall_',
translate=translate)
 wall_feats = [cc for cc in data_all.columns if 'wall_' in cc]
translate = {
'moscer': 'mosaic',
'cemento': 'cement',
'other': 'other',
'natur': 'natural',
'notiene': 'no_floor',
'madera': 'wood'
}
data_all = to_english(data_all, sp_pre='piso', eng_pre='floor_', translate=translate)
 floor_feats = [cc for cc in data_all.columns if 'floor_' in cc]
translate = {
'zinc': 'zinc',
'entrepiso': 'fiber cement',
'cane': 'natural fibers',
'otro': 'other'
}
data_all = to_english(data_all, sp_pre='techo', eng_pre='roof_', translate=translate)
 roof_feats = [cc for cc in data_all.columns if 'roof_' in cc] 
<define_variables>
combine.BsmtCond.value_counts()
material_feats = roof_feats + wall_feats + floor_feats<categorify>
combine['BsmtCond']=combine['BsmtCond'].fillna('None')
 combine.BsmtCond.value_counts()
translate = {
'guadentro': 'inside_house',
'guafuera': 'outside_house',
'guano': 'no'
}
data_all = to_english(data_all, sp_pre='abasta', eng_pre='water_provision_',
translate=translate)
 water_feats = [cc for cc in data_all.columns if 'water_provision_' in cc]
translate = {
'public': 'public',
'planpri': 'private_plan',
'noelec': 'no',
'coopele': 'cooperate'
}
data_all = to_english(data_all, sp_pre='', eng_pre='electric_', translate=translate)
 elec_feats = [cc for cc in data_all.columns if 'electric_' in cc]
translate = {
'cinar1': 'no',
'cinar2': 'electricity',
'cinar3': 'gas',
'cinar4': 'charcoal'
}
data_all = to_english(data_all, sp_pre='energco', eng_pre='energy_', translate=translate)
 energy_feats = [cc for cc in data_all.columns if 'energy_' in cc]
translate = {
'1': 'no',
'2': 'sewer',
'3': 'septic_tank',
'5': 'black hole',
'6': 'other'
}
data_all = to_english(data_all, sp_pre='sanitario', eng_pre='toilet_', translate=translate)
 toilet_feats = [cc for cc in data_all.columns if 'toilet_' in cc]
translate = {
'1': 'tanker truck',
'2': 'buried',
'3': 'burning',
'4': 'throw empty place',
'5': 'throw to river',
'6': 'other'
}
data_all = to_english(data_all, sp_pre='elimbasu', eng_pre='rubbish_', translate=translate)
 rubbish_feats = [cc for cc in data_all.columns if 'rubbish_' in cc]<define_variables>
combine.BsmtExposure.value_counts()
facility_feats = water_feats + elec_feats + energy_feats + toilet_feats + rubbish_feats<categorify>
combine['BsmtExposure']=combine['BsmtExposure'].fillna('None')
 combine.BsmtExposure.value_counts()
translate = {
'1': 'own_fully_paid',
'2': 'own_pay_installment',
'3': 'rented',
'4': 'precarious',
'5': 'other'
}
data_all = to_english(data_all, sp_pre='tipovivi', eng_pre='living_type_',
translate=translate )<define_variables>
combine.BsmtFinType1.value_counts()
live_feats = [cc for cc in data_all.columns if 'living_type_' in cc]<split>
combine['BsmtFinType1']=combine['BsmtFinType1'].fillna('None')
 combine.BsmtFinType1.value_counts()
head_df = data_all.query('parentesco1 == 1')
 train = head_df.loc[head_df['Target'].notnull() , :]
test = head_df.loc[head_df['Target'].isnull() , :]<define_variables>
combine.BsmtFinType1[combine.BsmtFinSF1.isnull() ]
features = basic_feats + house_head_feats + material_feats + facility_feats + live_feats + fe_feats + agg_feats
print('<split>
combine['BsmtFinSF1']=combine['BsmtFinSF1'].fillna(0)
 combine.BsmtFinSF1.isnull().sum()
X, y = train[['idhogar', 'Id'] + features], train['Target']
X_train, X_valid, y_train, y_valid = train_test_split(X[features], y,
test_size=0.1, random_state=0 )<define_search_space>
combine.BsmtFinType2.value_counts()
gbm = lgb.LGBMClassifier(n_jobs=4, random_state=0, class_weight='balanced')
 param_grid = {'num_leaves': np.arange(10, 50, 10),
'learning_rate': np.arange(0.05, 0.2, 0.05),
'n_estimators': np.arange(10, 25, 5)}
scoring = {'f1_macro': get_scorer('f1_macro')}
metric = 'f1_macro'<train_on_grid>
combine['BsmtFinType2']=combine['BsmtFinType2'].fillna('None')
 combine.BsmtFinType2.value_counts()
gs = GridSearchCV(gbm,
param_grid=param_grid,
scoring=scoring,
cv=5,
refit=metric,
verbose=True,
n_jobs=4, 
)
 gs.fit(X_train, y_train)
 best_estimator = gs.best_estimator_<import_modules>
combine['BsmtFinSF2']=combine['BsmtFinSF2'].fillna(0)
 combine.BsmtFinSF2.isnull().sum()
from sklearn.metrics import f1_score<predict_on_test>
combine['BsmtUnfSF']=combine['BsmtUnfSF'].fillna(0)
 combine.BsmtUnfSF.isnull().sum()
y_pred = best_estimator.predict(X_valid)
 macro_f1 = f1_score(y_valid, y_pred, average='macro')
 print(macro_f1 )<predict_on_test>
combine['TotalBsmtSF']=combine['TotalBsmtSF'].fillna(0)
 combine.TotalBsmtSF.isnull().sum()
X_test = test[['Id', 'idhogar'] + features]
y_pred = best_estimator.predict(X_test[features])
 pred_for_heads = pd.DataFrame({'Id': X_test['Id'], 'idhogar': X_test['idhogar'],
'Target': y_pred})
 pred_for_heads['Target'] = pred_for_heads['Target'].apply(int )<merge>
combine.Electrical.value_counts()
test_all = data_all.loc[data_all['Target'].isnull() , ['Id', 'idhogar']]
submit = pd.merge(test_all, pred_for_heads[['idhogar', 'Target']], how='left' )<count_missing_values>
combine.Neighborhood[combine.Electrical.isnull() ]
sum(submit['Target'].isnull() )<data_type_conversions>
combine[combine.Neighborhood == 'Timber'].Electrical.value_counts()
submit.fillna(1, inplace=True)
 submit['Target'] = submit['Target'].astype(np.int16)
 submit.head()<save_to_csv>
combine['Electrical']=combine['Electrical'].fillna('SBrkr')
 combine.Electrical.value_counts()
submit[['Id', 'Target']].to_csv('submisssion.csv', index=False )<set_options>
combine.BsmtFullBath.value_counts()
%matplotlib inline
warnings.filterwarnings("ignore" )<feature_engineering>
combine['BsmtFullBath']=combine['BsmtFullBath'].fillna(0)
 combine.BsmtFullBath.value_counts()
def do_features(df):
feats_div = [('children_fraction', 'r4t1', 'r4t3'), 
('working_man_fraction', 'r4h2', 'r4t3'), 
('all_man_fraction', 'r4h3', 'r4t3'), 
('human_density', 'tamviv', 'rooms'), 
('human_bed_density', 'tamviv', 'bedrooms'), 
('bed_density', 'bedrooms', 'rooms'), 
('rent_per_person', 'v2a1', 'r4t3'), 
('rent_per_room', 'v2a1', 'rooms'), 
('mobile_density', 'qmobilephone', 'r4t3'), 
('tablet_density', 'v18q1', 'r4t3'), 
('mobile_adult_density', 'qmobilephone', 'r4t2'), 
('tablet_adult_density', 'v18q1', 'r4t2'), 
('male_over_female', 'r4h3', 'r4m3'), 
('man12plus_over_women12plus', 'r4h2', 'r4m2'), 
('pesioner_over_working', 'hogar_mayor', 'hogar_adul'), 
('children_over_working', 'hogar_nin', 'hogar_adul'), 
('education_fraction', 'escolari', 'age')
 ]
feats_sub = [('people_not_living', 'tamhog', 'tamviv'), 
('non_bedrooms', 'rooms', 'bedrooms'), 
('people_weird_stat', 'tamhog', 'r4t3')]
for f_new, f1, f2 in feats_div:
df['fe_' + f_new] =(df[f1] / df[f2] ).astype(np.float32)
 for f_new, f1, f2 in feats_sub:
df['fe_' + f_new] =(df[f1] - df[f2] ).astype(np.float32)
 aggs_num = {'age': ['min', 'max', 'mean', 'count'],
'escolari': ['min', 'max', 'mean', 'std'],
'fe_education_fraction': ['min', 'max', 'mean', 'std']
}
aggs_cat = {'dis': ['mean']}
for s_ in ['estadocivil', 'parentesco', 'instlevel']:
for f_ in [f_ for f_ in df.columns if f_.startswith(s_)]:
aggs_cat[f_] = ['mean']
for name_, df_ in [('18', df.query('age >= 18')) ]:
df_agg = df_.groupby('idhogar' ).agg({**aggs_num, **aggs_cat} ).astype(np.float32)
 df_agg.columns = pd.Index(['agg' + name_ + '_' + e[0] + "_" + e[1].upper() for e in df_agg.columns.tolist() ])
 df = df.join(df_agg, how='left', on='idhogar')
 del df_agg
df.drop([f_ for f_ in df.columns if f_.startswith('SQB')or f_ == 'agesq'], axis=1, inplace=True)
 df.drop(['Id'], axis=1, inplace=True)
 df.drop(['hhsize', 'female', 'area2'], axis=1, inplace=True)
 return df<categorify>
combine.BsmtHalfBath.value_counts()
def convert_OHE2LE(df):
tmp_df = df.copy(deep=True)
 for s_ in ['pared', 'piso', 'techo', 'abastagua', 'sanitario', 'energcocinar', 'elimbasu',
'epared', 'etecho', 'eviv', 'estadocivil', 'parentesco',
'instlevel', 'lugar', 'tipovivi',
'manual_elec']:
if 'manual_' not in s_:
cols_s_ = [f_ for f_ in df.columns if f_.startswith(s_)]
elif 'elec' in s_:
cols_s_ = ['public', 'planpri', 'noelec', 'coopele']
sum_ohe = tmp_df[cols_s_].sum(axis=1 ).unique()
if 0 in sum_ohe:
print('The OHE in {} is incomplete.A new column will be added before label encoding' 
.format(s_))
col_dummy = s_+'_dummy'
tmp_df[col_dummy] =(tmp_df[cols_s_].sum(axis=1)== 0 ).astype(np.int8)
 cols_s_.append(col_dummy)
 sum_ohe = tmp_df[cols_s_].sum(axis=1 ).unique()
if 0 in sum_ohe:
print("The category completion did not work")
 tmp_cat = tmp_df[cols_s_].idxmax(axis=1)
 tmp_df[s_ + '_LE'] = LabelEncoder().fit_transform(tmp_cat ).astype(np.int16)
 if 'parentesco1' in cols_s_:
cols_s_.remove('parentesco1')
 tmp_df.drop(cols_s_, axis=1, inplace=True)
 return tmp_df<load_from_csv>
combine['BsmtHalfBath']=combine['BsmtHalfBath'].fillna(0)
 combine.BsmtHalfBath.value_counts()
train = pd.read_csv('.. /input/train.csv', nrows=None)
 test = pd.read_csv('.. /input/test.csv' )<categorify>
combine.KitchenQual.value_counts()
def process_df(df_):
encode_data(df_)
 for f_ in ['v2a1', 'v18q1', 'meaneduc', 'SQBmeaned']:
df_[f_] = df_[f_].fillna(0)
 df_['rez_esc'] = df_['rez_esc'].fillna(-1)
 return do_features(df_)
 train = process_df(train)
 test = process_df(test )<concatenate>
combine.KitchenAbvGr.value_counts()
def train_test_apply_func(train_, test_, func_):
test_['Target'] = 0
xx = pd.concat([train_, test_])
 xx_func = func_(xx)
 train_ = xx_func.iloc[:train_.shape[0], :]
test_ = xx_func.iloc[train_.shape[0]:, :].drop('Target', axis=1)
 del xx, xx_func
return train_, test_<feature_engineering>
combine.KitchenAbvGr[combine.KitchenQual.isnull() ]
train, test = train_test_apply_func(train, test, convert_OHE2LE )<categorify>
combine.KitchenQual[combine.KitchenAbvGr == 1].value_counts()
cols_2_ohe = ['eviv_LE', 'etecho_LE', 'epared_LE', 'elimbasu_LE',
'energcocinar_LE', 'sanitario_LE', 'manual_elec_LE',
'pared_LE']
cols_nums = ['age', 'meaneduc', 'dependency',
'hogar_nin', 'hogar_adul', 'hogar_mayor', 'hogar_total',
'bedrooms', 'overcrowding']
def convert_geo2aggs(df_):
tmp_df = pd.concat([df_[(['lugar_LE', 'idhogar']+cols_nums)],
pd.get_dummies(df_[cols_2_ohe],
columns=cols_2_ohe)],axis=1)
 geo_agg = tmp_df.groupby(['lugar_LE','idhogar'] ).mean().groupby('lugar_LE' ).mean().astype(np.float32)
 geo_agg.columns = pd.Index(['geo_' + e + '_MEAN' for e in geo_agg.columns.tolist() ])
 del tmp_df
return df_.join(geo_agg, how='left', on='lugar_LE')
 train, test = train_test_apply_func(train, test, convert_geo2aggs )<prepare_x_and_y>
combine['KitchenQual']=combine['KitchenQual'].fillna('TA')
 combine.KitchenQual.value_counts()
X = train.query('parentesco1==1')
 y = X['Target'] - 1
X = X.drop(['Target'], axis=1 )<define_variables>
combine.Functional.value_counts()
cols_2_drop = ['abastagua_LE', 'agg18_estadocivil1_MEAN', 'agg18_instlevel6_MEAN', 'agg18_parentesco10_MEAN', 'agg18_parentesco11_MEAN', 'agg18_parentesco12_MEAN', 'agg18_parentesco4_MEAN', 'agg18_parentesco5_MEAN', 'agg18_parentesco6_MEAN', 'agg18_parentesco7_MEAN', 'agg18_parentesco8_MEAN', 'agg18_parentesco9_MEAN', 'fe_people_not_living', 'fe_people_weird_stat', 'geo_elimbasu_LE_3_MEAN', 'geo_elimbasu_LE_4_MEAN', 'geo_energcocinar_LE_0_MEAN', 'geo_energcocinar_LE_1_MEAN', 'geo_energcocinar_LE_2_MEAN', 'geo_epared_LE_0_MEAN', 'geo_epared_LE_2_MEAN', 'geo_etecho_LE_2_MEAN', 'geo_eviv_LE_0_MEAN', 'geo_hogar_mayor_MEAN', 'geo_hogar_nin_MEAN', 'geo_manual_elec_LE_1_MEAN', 'geo_manual_elec_LE_2_MEAN', 'geo_manual_elec_LE_3_MEAN', 'geo_pared_LE_0_MEAN', 'geo_pared_LE_1_MEAN', 'geo_pared_LE_3_MEAN', 'geo_pared_LE_4_MEAN', 'geo_pared_LE_5_MEAN', 'geo_pared_LE_6_MEAN', 'geo_pared_LE_7_MEAN', 'hacapo', 'hacdor', 'mobilephone', 'parentesco1', 'parentesco_LE', 'rez_esc', 'techo_LE', 'v14a', 'v18q']
X.drop(( cols_2_drop+['idhogar']), axis=1, inplace=True)
 test.drop(( cols_2_drop+['idhogar']), axis=1, inplace=True )<compute_train_metric>
combine['Functional']=combine['Functional'].fillna('Typ')
 combine.Functional.value_counts()
def evaluate_macroF1_lgb(truth, predictions):
pred_labels = predictions.reshape(len(np.unique(truth)) ,-1 ).argmax(axis=0)
 f1 = f1_score(truth, pred_labels, average='macro')
 return('macroF1', f1, True)
 def learning_rate_power_0997(current_iter):
base_learning_rate = 0.1
min_learning_rate = 0.02
lr = base_learning_rate * np.power (.99, current_iter)
 return max(lr, min_learning_rate)
 fit_params={"early_stopping_rounds":300,
"eval_metric" : 'multiclass',
"eval_metric" : evaluate_macroF1_lgb,
'eval_names': ['train', 'early_stop'],
'callbacks': [lgb.reset_parameter(learning_rate=learning_rate_power_0997)],
'verbose': False,
'categorical_feature': 'auto'} 
<init_hyperparams>
combine.FireplaceQu.value_counts()
opt_parameters = {'colsample_bytree': 0.89, 'min_child_samples': 90, 'num_leaves': 14, 'subsample': 0.96}<categorify>
combine.Fireplaces.value_counts()
class VotingPrefitClassifier(VotingClassifier):

 def fit(self, X, y, sample_weight=None, **fit_params):
self.estimators_ = [x[1] for x in self.estimators]
self.le_ = LabelEncoder().fit(y)
 self.classes_ = self.le_.classes_
return self<train_on_grid>
combine.FireplaceQu.isnull().sum()
def train_single_model(clf_, X_, y_, random_state_=314, opt_parameters_={}, fit_params_={}):
c = clone(clf_)
 c.set_params(**opt_parameters_)
 c.set_params(random_state=random_state_)
 return c.fit(X_, y_, **fit_params_)
 def train_model_in_nestedCV(model, X, y, metric, metric_args={},
model_name='xmodel',
inner_seed=31416, inner_n=10, outer_seed=314, outer_n=10,
opt_parameters_={}, fit_params_={},
verbose=True):
clfs = []
perf_eval = {'score_i_oof': [],
'score_i_ave': [],
'score_i_std': [],
'score_i_early_stop_ave': [],
'score_o_early_stop': [],
'score_o_early_stop_vc_w0_soft': [],
'score_o_early_stop_vc_w0_hard': []
}
y_full_oof = pd.Series(np.zeros(shape=(y.shape[0],)) ,
index=y.index)
 if 'sample_weight' in metric_args:
sample_weight=metric_args['sample_weight']
outer_cv = StratifiedKFold(outer_n, shuffle=True, random_state=outer_seed)
 for n_outer_fold,(outer_trn_idx, outer_val_idx)in enumerate(outer_cv.split(X,y)) :
print('--- Outer loop iteration: {} ---'.format(n_outer_fold))
X_out, y_out = X.iloc[outer_trn_idx], y.iloc[outer_trn_idx]
X_stp, y_stp = X.iloc[outer_val_idx], y.iloc[outer_val_idx]
inner_cv = StratifiedKFold(inner_n, shuffle=True, random_state=inner_seed+n_outer_fold)
 y_outer_oof = pd.Series(np.zeros(shape=(X_out.shape[0],)) ,
index=X_out.index)
 scores_inner = []
clfs_inner = []
for n_inner_fold,(inner_trn_idx, inner_val_idx)in enumerate(inner_cv.split(X_out,y_out)) :
X_trn, y_trn = X_out.iloc[inner_trn_idx], y_out.iloc[inner_trn_idx]
X_val, y_val = X_out.iloc[inner_val_idx], y_out.iloc[inner_val_idx]
if fit_params_:
fit_params_["eval_set"] = [(X_trn,y_trn),(X_stp,y_stp)]
fit_params_['verbose'] = False
clf = train_single_model(model, X_trn, y_trn, 314+n_inner_fold, opt_parameters_, fit_params_)
 clfs_inner.append(( '{}{}_inner'.format(model_name,n_inner_fold), clf))
y_outer_oof.iloc[inner_val_idx] = clf.predict(X_val)
 if 'sample_weight' in metric_args:
metric_args['sample_weight'] = y_val.map(sample_weight)
 scores_inner.append(metric(y_val, y_outer_oof.iloc[inner_val_idx], **metric_args))
del X_trn, y_trn, X_val, y_val
if 'sample_weight' in metric_args:
metric_args['sample_weight'] = y_outer_oof.map(sample_weight)
 perf_eval['score_i_oof'].append(metric(y_out, y_outer_oof, **metric_args))
perf_eval['score_i_ave'].append(np.mean(scores_inner))
perf_eval['score_i_std'].append(np.std(scores_inner))
if 'sample_weight' in metric_args:
metric_args['sample_weight'] = y_stp.map(sample_weight)
 score_inner_early_stop = [metric(y_stp, clf_.predict(X_stp), **metric_args)
 for _,clf_ in clfs_inner]
perf_eval['score_i_early_stop_ave'].append(np.mean(score_inner_early_stop))
w = np.array(scores_inner)
 for w_, w_name_ in [(None, '_w0')
 ]:
vc = VotingPrefitClassifier(clfs_inner, weights=w_ ).fit(X_stp, y_stp)
 for vote_type in ['soft', 'hard']:
vc.voting = vote_type
if 'sample_weight' in metric_args:
metric_args['sample_weight'] = y_stp.map(sample_weight)
 perf_eval['score_o_early_stop_vc{}_{}'.format(w_name_, vote_type)].append(metric(y_stp, vc.predict(X_stp), **metric_args))
if fit_params_:
fit_params_["eval_set"] = [(X_out,y_out),(X_stp,y_stp)]
if verbose:
fit_params_['verbose'] = 200
clf = train_single_model(model, X_out, y_out, 314+n_outer_fold, opt_parameters_, fit_params_)
 if 'sample_weight' in metric_args:
metric_args['sample_weight'] = y_stp.map(sample_weight)
 perf_eval['score_o_early_stop'].append(metric(y_stp, clf.predict(X_stp), **metric_args))
clfs.append(( '{}{}'.format(model_name,n_outer_fold), clf))
y_full_oof.iloc[outer_val_idx] = clf.predict(X_stp)
 del inner_cv, X_out, y_out, X_stp, y_stp, clfs_inner
return clfs, perf_eval, y_full_oof
def print_nested_perf_clf(name, perf_eval):
print('Performance of the inner-loop model(the two should agree):')
 print(' Mean(mean(Val)) score inner {} Classifier: {:.4f}+-{:.4f}'.format(name,
np.mean(perf_eval['score_i_ave']),
np.std(perf_eval['score_i_ave'])
))
print(' Mean(mean(EarlyStop)) score inner {} Classifier: {:.4f}+-{:.4f}'.format(name,
np.mean(perf_eval['score_i_early_stop_ave']),
np.std(perf_eval['score_i_early_stop_ave'])
))
print('Mean(inner OOF)score inner {} Classifier: {:.4f}+-{:.4f}'.format(name,
np.mean(perf_eval['score_i_oof']),
np.std(perf_eval['score_i_oof'])
))
print('Mean(EarlyStop)score outer {} Classifier: {:.4f}+-{:.4f}'.format(name,
np.mean(perf_eval['score_o_early_stop']),
np.std(perf_eval['score_o_early_stop'])
))
print('Mean(EarlyStop)outer VotingPrefit SOFT: {:.4f}+-{:.4f}'.format(np.mean(perf_eval['score_o_early_stop_vc_w0_soft']),
np.std(perf_eval['score_o_early_stop_vc_w0_soft'])
))
print('Mean(EarlyStop)outer VotingPrefit HARD: {:.4f}+-{:.4f}'.format(np.mean(perf_eval['score_o_early_stop_vc_w0_hard']),
np.std(perf_eval['score_o_early_stop_vc_w0_hard'])
))<train_on_grid>
combine['FireplaceQu']=combine['FireplaceQu'].fillna('None')
 combine.FireplaceQu.value_counts()
clf = lgb.LGBMClassifier(max_depth=-1, learning_rate=0.1, objective='multiclass',
random_state=1, silent=True, metric='None',
n_jobs=4, n_estimators=5000, class_weight='balanced')
 clfs_, perf_eval, y_full_oof = train_model_in_nestedCV(clf, X, y, f1_score,
metric_args={'average':'macro'},
model_name='lgbm',
opt_parameters_=opt_parameters,
fit_params_=fit_params,
inner_n=10, outer_n=10,
verbose=False )<train_on_grid>
combine.GarageType.value_counts()
w = np.array(perf_eval['score_o_early_stop'])
 ws = [(None, '_w0'), 
(w/w.sum() , '_w1'), 
(( w**2)/np.sum(w**2), '_w2')
 ]
vc = {}
for w_, w_name_ in ws:
vc['vc{}'.format(w_name_)] = VotingPrefitClassifier(clfs_, weights=w_ ).fit(X, y)
 clf_final = clfs_[0][1]<compute_test_metric>
combine['GarageType']=combine['GarageType'].fillna('None')
 combine.GarageType.value_counts()
global_score = np.mean(perf_eval['score_i_oof'])
 global_score_std = np.std(perf_eval['score_i_oof'])
 print_nested_perf_clf('lgbm', perf_eval)
 print('Outer OOF score {} Classifier: {:.4f}'.format('lgbm', f1_score(y, y_full_oof, average='macro')) )<create_dataframe>
combine['GarageYrBlt']=combine['GarageYrBlt'].fillna(0)
 combine.GarageYrBlt.isnull().sum()
perf_eval_df = pd.DataFrame(perf_eval)
 perf_eval_df<import_modules>
combine.GarageFinish.value_counts()
from sklearn.metrics import precision_score, recall_score, classification_report<feature_engineering>
combine['GarageFinish']=combine['GarageFinish'].fillna('None')
 combine.GarageFinish.value_counts()
shap_values = shap.TreeExplainer(clf_final.booster_ ).shap_values(X)
<load_from_csv>
combine.GarageCars.value_counts()
y_subm = pd.read_csv('.. /input/sample_submission.csv' )<save_to_csv>
combine['GarageCars']=combine['GarageCars'].fillna(0)
 combine.GarageCars.value_counts()
now = datetime.now()
sub_file = 'submission_LGB_{:.4f}_{}.csv'.format(global_score, str(now.strftime('%Y-%m-%d-%H-%M')))
 y_subm['Target'] = clf_final.predict(test)+ 1
y_subm.to_csv(sub_file, index=False)
 for vc_name_,vc_ in vc.items() :
for vc_type_ in ['soft', 'hard']:
vc_.voting = vc_type_
name = '{}_{}'.format(vc_name_, vc_type_)
 y_subm_vc = y_subm.copy(deep=True)
 y_subm_vc.loc[:,'Target'] = vc_.predict(test)+ 1
sub_file = 'submission_{}_LGB_{:.4f}_{}.csv'.format(name,
global_score,
str(now.strftime('%Y-%m-%d-%H-%M')) 
)
 y_subm_vc.to_csv(sub_file, index=False )<load_from_csv>
combine['GarageArea']=combine['GarageArea'].fillna(0)
 combine.GarageArea.isnull().sum()
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 test_Id = test_df['Id']<create_dataframe>
combine.GarageQual.value_counts()
def preprocess(train_df, test_df, dropna = False, drop_obtype = True, replace_nan_with = 0, return_with_top100 = False, lgbfactor = 0):
train_df.drop(columns = ['Id'],inplace = True, errors="ignore")
 test_df.drop(columns = ['Id'], inplace = True, errors = "ignore")
 nanvalues = train_df.isnull().sum(axis = 0 ).values
nullvalues = pd.DataFrame({"Column":train_df.columns, "Count":nanvalues})
 nullvalues.sort_values(by=["Count"], ascending = False, inplace = True)
 nancols = nullvalues[nullvalues['Count']>0]["Column"].values.tolist()
objcols = train_df[train_df.columns[(train_df.dtypes == 'object' ).values]].columns.values.tolist()
if(drop_obtype == True):
train_df.drop(columns = objcols,inplace = True)
 test_df.drop(columns = objcols,inplace = True)
 if(dropna == True):
train_df.drop(columns = nancols, inplace = True)
 test_df.drop(columns = nancols, inplace = True)
 if(dropna == False):
train_df.replace(np.nan, replace_nan_with, inplace = True)
 test_df.replace(np.nan, replace_nan_with, inplace = True)
 labels = train_df['Target']
train_df.drop(columns = ['Target'],inplace = True, errors="ignore")
 rfc = RandomForestClassifier()
rfc.fit(train_df, labels)
 scores = rfc.feature_importances_.tolist()
report = pd.DataFrame({"feature":train_df.columns, "score":scores})
 report.sort_values(by = ["score"],ascending = False, inplace = True)
 top100cols = report.head(100)['feature'].values.tolist()
if(return_with_top100 == True):
train_df = train_df[top100cols]
test_df = test_df[top100cols]
class_weights = class_weight.compute_class_weight('balanced',
np.unique(labels),
labels)
 class_weights = dict(zip(np.unique(labels.values-lgbfactor), class_weights))
return train_df, test_df, labels, nancols, objcols, report, top100cols, class_weights 
<create_dataframe>
combine['GarageQual']=combine['GarageQual'].fillna('None')
 combine.GarageQual.value_counts()
train, test, labels, nancols,objcols, report, top100cols, class_weightss = preprocess(train_df.copy() , test_df.copy() ,dropna = False, lgbfactor = 1 )<import_modules>
combine.GarageCond.value_counts()
import lightgbm as lgb
import sklearn.model_selection as model_selection
from sklearn.metrics import f1_score, make_scorer<choose_model_class>
combine['GarageCond']=combine['GarageCond'].fillna('None')
 combine.GarageCond.value_counts()
lgmodel = lgb.LGBMClassifier(class_weight=class_weightss, metric = "multi_logloss",num_class = 4)
<compute_train_metric>
combine.PoolArea.value_counts()
def get_score(model, train, label, fold):
score = model_selection.cross_val_score(model, train , label, cv = fold, scoring = make_scorer(f1_score, average = "macro"))
return score.mean()<find_best_params>
combine.PoolQC.value_counts()
kf = model_selection.KFold(n_splits=5, shuffle=True,random_state=2017)
 templabel = labels.loc[train['parentesco1']==1]
temptrain = train[train['parentesco1']==1]
heads_scores = get_score(lgmodel, temptrain, templabel-1, kf)
 overall_scores = get_score(lgmodel, train, labels-1, kf)
 print("Heads mean score:",heads_scores.mean())
 print("Overall mean score:",overall_scores.mean() )<predict_on_test>
combine['PoolQC']=combine['PoolQC'].fillna('None')
 combine.PoolQC.value_counts()
lgmodel.fit(train, labels-1)
 lgpreds = lgmodel.predict(test )<save_to_csv>
combine.Fence.value_counts()
sub_df = pd.DataFrame({"Id":test_Id.values})
 sub_df["Target"] = lgpreds + 1
sub_df.to_csv("LightGBM.csv", index=False )<set_options>
combine['Fence']=combine['Fence'].fillna('None')
 combine.Fence.value_counts()
%matplotlib inline
%reload_ext autoreload
%autoreload 2
np.set_printoptions(threshold=50, edgeitems=20)
 pd.set_option('display.max_rows', 500)
 pd.set_option('display.max_columns', 500)
<load_from_csv>
combine.MiscFeature.value_counts()
PATH = '.. /input/'
train_df = pd.read_csv(PATH+'train.csv')
 test_df = pd.read_csv(PATH+'test.csv' )<define_variables>
combine['MiscFeature']=combine['MiscFeature'].fillna('None')
 combine.MiscFeature.value_counts()
cat_vars = ['hacdor',
'hacapo',
'v14a',
'refrig',
'v18q',
'paredblolad',
'paredzocalo',
'paredpreb',
'pareddes',
'paredmad',
'paredzinc',
'paredfibras',
'paredother',
'pisomoscer',
'pisocemento',
'pisoother',
'pisonatur',
'pisonotiene',
'pisomadera',
'techozinc',
'techoentrepiso',
'techocane',
'techootro',
'cielorazo',
'abastaguadentro',
'abastaguafuera',
'abastaguano',
'public',
'planpri',
'noelec',
'coopele',
'sanitario1',
'sanitario2',
'sanitario3',
'sanitario5',
'sanitario6',
'energcocinar1',
'energcocinar2',
'energcocinar3',
'energcocinar4',
'elimbasu1',
'elimbasu2',
'elimbasu3',
'elimbasu4',
'elimbasu5',
'elimbasu6',
'epared1',
'epared2',
'epared3',
'etecho1',
'etecho2',
'etecho3',
'eviv1',
'eviv2',
'eviv3',
'dis',
'estadocivil1',
'estadocivil2',
'estadocivil3',
'estadocivil4',
'estadocivil5',
'estadocivil6',
'estadocivil7',
'parentesco1',
'parentesco2',
'parentesco3',
'parentesco4',
'parentesco5',
'parentesco6',
'parentesco7',
'parentesco8',
'parentesco9',
'parentesco10',
'parentesco11',
'parentesco12',
'dependency',
'edjefe',
'edjefa',
'instlevel1',
'instlevel2',
'instlevel3',
'instlevel4',
'instlevel5',
'instlevel6',
'instlevel7',
'instlevel8',
'instlevel9',
'tipovivi1',
'tipovivi2',
'tipovivi3',
'tipovivi4',
'tipovivi5',
'computer',
'television',
'mobilephone',
'lugar1',
'lugar2',
'lugar3',
'lugar4',
'lugar5',
'lugar6',
'area1',
'area2']
contin_vars = ['v2a1',
'rooms',
'v18q1',
'r4h1',
'r4h2',
'r4h3',
'r4m1',
'r4m2',
'r4m3',
'r4t1',
'r4t2',
'r4t3',
'tamhog',
'escolari',
'rez_esc',
'hhsize',
'male',
'female',
'hogar_nin',
'hogar_adul',
'hogar_mayor',
'hogar_total',
'meaneduc',
'bedrooms',
'overcrowding',
'qmobilephone',
'age',
'SQBescolari',
'SQBage',
'SQBhogar_total',
'SQBedjefe',
'SQBhogar_nin',
'SQBovercrowding',
'SQBdependency',
'SQBmeaned',
'agesq']
objective= ['Target']
ids = ['Id']<data_type_conversions>
combine.SaleType.value_counts()
for v in cat_vars: train_df[v] = train_df[v].astype('category' ).cat.as_ordered()
apply_cats(test_df, train_df )<data_type_conversions>
combine['SaleType']=combine['SaleType'].fillna('WD')
 combine.SaleType.value_counts()
for v in contin_vars:
train_df[v] = train_df[v].fillna(0 ).astype('float32')
 test_df[v] = test_df[v].fillna(0 ).astype('float32' )<create_dataframe>
combine['SalePrice']=combine['SalePrice'].fillna(0)
 combine.SalePrice.isnull().sum()
for dataframe in(train_df, test_df):
dataframe['Rent_per_individual'] = dataframe['v2a1']/dataframe['r4t3']
dataframe['Rent_per_child'] = dataframe['v2a1']/dataframe['r4t1']
dataframe['Rent_per_over_65'] = dataframe['v2a1']/dataframe['r4t3']
dataframe['Rent_per_room'] = dataframe['v2a1']/dataframe['rooms']
dataframe['Rent_per_bedrooms'] = dataframe['v2a1']/dataframe['bedrooms']
dataframe['Proportion_under_12'] = dataframe['r4t1']/dataframe['r4t3']
dataframe['Proportion_under_12_male'] = dataframe['r4h1']/dataframe['r4t3']
dataframe['Proportion_under_12_female'] = dataframe['r4m1']/dataframe['r4t3']
dataframe['Proportion_male'] = dataframe['r4h3']/dataframe['r4t3']
dataframe['Proportion_female'] = dataframe['r4m3']/dataframe['r4t3']
dataframe['Rooms_per_individual'] = dataframe['rooms']/dataframe['r4t3']
dataframe['Rooms_per_child'] = dataframe['rooms']/dataframe['r4t1']
dataframe['Tablets_per_individual'] = dataframe['v18q1']/dataframe['r4t3']
dataframe['Tablets_per_child'] = dataframe['v18q1']/dataframe['r4t1']
dataframe['Years_schooling_per_individual'] = dataframe['escolari']/dataframe['r4t3']
dataframe['Years_schooling_per_adult'] = dataframe['escolari']/(dataframe['r4t3']-dataframe['r4t1'])
 dataframe['Years_schooling_per_child'] = dataframe['escolari']/dataframe['r4t1']
dataframe['Proportion_under_19'] = dataframe['hogar_nin']/dataframe['r4t3']
dataframe['Proportion_over_19'] = dataframe['hogar_adul']/dataframe['r4t3']
dataframe['Proportion_under_65'] =(dataframe['hogar_total']-dataframe['hogar_mayor'])/dataframe['r4t3']
dataframe['Proportion_over_65'] = dataframe['hogar_mayor']/dataframe['r4t3']
dataframe['Bedrooms_per_individual'] = dataframe['bedrooms']/dataframe['r4t3']
dataframe['Bedrooms_per_child'] = dataframe['bedrooms']/dataframe['r4t1']
dataframe['Bedrooms_per_over_65'] = dataframe['bedrooms']/dataframe['r4t3']
dataframe['Extreme_conditions_flag'] =(dataframe['abastaguano'] & dataframe['noelec'] & dataframe['sanitario1'] & dataframe['energcocinar1'])
 dataframe['bedrooms_to_rooms'] = dataframe['bedrooms']/dataframe['rooms']
dataframe['tamhog_to_rooms'] = dataframe['tamhog']/dataframe['rooms']
dataframe['tamhog_to_bedrooms'] = dataframe['tamhog']/dataframe['bedrooms']
dataframe['r4t3_to_tamhog'] = dataframe['r4t3']/dataframe['tamhog']
dataframe['hhsize_to_rooms'] = dataframe['hhsize']/dataframe['rooms']
dataframe['hhsize_to_bedrooms'] = dataframe['hhsize']/dataframe['bedrooms']
dataframe['rent_to_hhsize'] = dataframe['v2a1']/dataframe['hhsize']
dataframe['qmobilephone_to_r4t3'] = dataframe['qmobilephone']/dataframe['r4t3']
dataframe['qmobilephone_to_v18q1'] = dataframe['qmobilephone']/dataframe['v18q1']<categorify>
combine.isnull().sum().sum()
groupper = train_df.groupby('idhogar')
 interactions =(pd.DataFrame(dict(
 head_partner_escolari=train_df.parentesco2.astype(int)* train_df.escolari.astype(int)))
.groupby(train_df.idhogar)
.max())
 my_features =(groupper.mean() [['escolari', 'age', 'male', 'meaneduc', 'SQBdependency']] 
.join(groupper.std() [['escolari', 'age']],
rsuffix='_std')
.join(groupper[['escolari', 'age']].min() , rsuffix="_min")
.join(groupper[['escolari', 'age']].max() , rsuffix="_max")
.join(groupper[['dis']].sum())
.join(interactions)
.join(groupper[['computer', 'television', 'qmobilephone', 'v18q1']].mean().sum(axis=1 ).rename('technics')))
 my_features.rename(columns={'escolari': 'escolari_mean', 'age': 'age_mean', 'male': 'male_mean', 'dis': 'dis_sum',
'meaneduc': 'meaneduc_mean', 'SQBdependency': 'SQBdependency_mean'}, inplace=True)
 train_df = train_df.merge(my_features, how='left', left_on='idhogar', right_on='idhogar' )<categorify>
combine.YrSold.value_counts()
groupper = test_df.groupby('idhogar')
 interactions =(pd.DataFrame(dict(
 head_partner_escolari=test_df.parentesco2.astype(int)* test_df.escolari.astype(int)))
.groupby(test_df.idhogar)
.max())
 my_features =(groupper.mean() [['escolari', 'age', 'male', 'meaneduc', 'SQBdependency']] 
.join(groupper.std() [['escolari', 'age']],
rsuffix='_std')
.join(groupper[['escolari', 'age']].min() , rsuffix="_min")
.join(groupper[['escolari', 'age']].max() , rsuffix="_max")
.join(groupper[['dis']].sum())
.join(interactions)
.join(groupper[['computer', 'television', 'qmobilephone', 'v18q1']].mean().sum(axis=1 ).rename('technics')))
 my_features.rename(columns={'escolari': 'escolari_mean', 'age': 'age_mean', 'male': 'male_mean', 'dis': 'dis_sum',
'meaneduc': 'meaneduc_mean', 'SQBdependency': 'SQBdependency_mean'}, inplace=True)
 test_df = test_df.merge(my_features, how='left', left_on='idhogar', right_on='idhogar' )<drop_column>
test_raw.YrSold.value_counts()
train_df.drop('idhogar', axis=1, inplace=True)
 test_df.drop('idhogar', axis=1, inplace=True )<define_variables>
combine['Age']=combine['YrSold']-combine['YearBuilt']
combine['PriceFlux']=2011-combine['YrSold']
combine['Renew']=combine['YrSold']-combine['YearRemodAdd']
combine['MSSubClass']=combine['MSSubClass'].apply(str )
new_features = ['Rent_per_individual',
'Rent_per_child',
'Rent_per_over_65',
'Rent_per_room',
'Rent_per_bedrooms',
'Proportion_under_12',
'Proportion_under_12_male',
'Proportion_under_12_female',
'Proportion_male',
'Proportion_female',
'Rooms_per_individual',
'Rooms_per_child',
'Tablets_per_individual',
'Tablets_per_child',
'Years_schooling_per_individual',
'Years_schooling_per_adult',
'Years_schooling_per_child',
'Proportion_under_19',
'Proportion_over_19',
'Proportion_under_65',
'Proportion_over_65',
'Bedrooms_per_individual',
'Bedrooms_per_child',
'Bedrooms_per_over_65',
'Extreme_conditions_flag',
'bedrooms_to_rooms',
'tamhog_to_rooms',
'tamhog_to_bedrooms',
'r4t3_to_tamhog',
'hhsize_to_rooms',
'hhsize_to_bedrooms',
'rent_to_hhsize',
'qmobilephone_to_r4t3',
'qmobilephone_to_v18q1',
'escolari_mean',
'age_mean',
'male_mean',
'meaneduc_mean',
'SQBdependency_mean',
'escolari_std',
'age_std',
'escolari_min',
'age_min',
'escolari_max',
'age_max',
'dis_sum',
'head_partner_escolari',
'technics']
for v in new_features:
train_df[v] = train_df[v].replace([np.inf, -np.inf], np.nan ).fillna(0 ).astype('float32')
 test_df[v] = test_df[v].replace([np.inf, -np.inf], np.nan ).fillna(0 ).astype('float32' )<prepare_x_and_y>
Label_cols=['MSSubClass','LotShape','LandContour','LandSlope','BldgType','HouseStyle','ExterQual','ExterCond','BsmtQual','BsmtCond','BsmtExposure','BsmtFinType1','BsmtFinType2','HeatingQC','CentralAir','Electrical','KitchenQual','Functional','FireplaceQu','GarageFinish','GarageQual','GarageCond','PavedDrive','PoolQC','Fence','MiscFeature','SaleType','SaleCondition']
for col in Label_cols:
lbe=LabelEncoder()
lbe.fit(list(combine[col].values))
combine[col]=lbe.transform(list(combine[col].values))
train_df = train_df.set_index('Id')
 train_df = train_df[cat_vars+contin_vars+new_features+objective]
df, y, nas, mapper = proc_df(train_df, 'Target', do_scale=True )<categorify>
combine=combine.drop(['YearBuilt','YearRemodAdd','MoSold','YrSold','GarageYrBlt','GarageArea','TotRmsAbvGrd'],axis=1)
 combine.shape
test_df = test_df.set_index('Id')
 test_df['Target'] = 0
test_df = test_df[cat_vars+contin_vars+new_features+['Target']]
df_test, _, nas, mapper = proc_df(test_df, 'Target', do_scale=True,
mapper=mapper, na_dict=nas )<choose_model_class>
combine=pd.get_dummies(combine)
 combine.shape
k = 5
kf = StratifiedKFold(n_splits=k, shuffle=True, random_state=1 )<split>
pd.options.display.max_columns=None
combine.columns.values
predicts = []
test_X = np.array(df_test)
 for train_index, test_index in kf.split(df, y):
print("
X_train, X_val = np.array(df)[train_index], np.array(df)[test_index]
y_train, y_val = y[train_index], y[test_index]
xgb_params = {
'learning_rate': 0.1,
'n_estimators': 1000,
'objective': 'multi:softmax',
'eval_metric': 'merror',
'silent': 1,
'num_class': 5,
'seed': 27}
d_train = xgb.DMatrix(X_train, y_train)
 d_valid = xgb.DMatrix(X_val, y_val)
 d_test = xgb.DMatrix(test_X)
 model = xgb.train(xgb_params, d_train, num_boost_round = 10000, evals=[(d_valid, 'eval')], verbose_eval=100,
early_stopping_rounds=50)
 xgb_pred = model.predict(d_test)
 predicts.append(list(xgb_pred))<define_variables>
test_raw.GarageType.value_counts()
preds=[]
for i in range(len(predicts[0])) :
sum=0
for j in range(k):
sum+=predicts[j][i]
preds.append(sum / k )<data_type_conversions>
combine=combine.drop(['MSZoning_None','Street_Pave','Alley_None','LotConfig_Inside','Neighborhood_NAmes','Condition1_Norm','Condition2_Norm','RoofStyle_Gable','RoofMatl_CompShg','Exterior1st_VinylSd','Exterior2nd_VinylSd','MasVnrType_None','Foundation_PConc','Heating_GasA','GarageType_None'],axis=1)
 combine.shape
df_test['Target'] = np.array(preds ).astype(int)
 sub = df_test[['Target']].copy()<save_to_csv>
train=train.drop(['Id'],axis=1 )
sub.to_csv('sample_submission.csv', index=False )<set_options>
test=combine[combine.SalePrice==0]
test=test.drop(['Id','SalePrice'],axis=1)
 test.shape
os.environ["SEED"] = "420" 
<load_from_csv>
def root_mean_squared_log_error(y_valid, y_preds):

 if len(y_preds)!=len(y_valid): return 'error_mismatch'
y_preds_new = [math.log(x)for x in y_preds]
y_valid_new = [math.log(x)for x in y_valid]
return mean_squared_error(y_valid_new, y_preds_new, squared=False )
df_train = pd.read_csv("/kaggle/input/gapvalidation/gap-test.tsv", delimiter="\t")
 df_val = pd.read_csv("/kaggle/input/gapvalidation/gap-validation.tsv", delimiter="\t")
 df_test = pd.read_csv("/kaggle/input/gapvalidation/gap-development.tsv", delimiter="\t")
 test_2 = pd.read_csv("/kaggle/input/gendered-pronoun-resolution/test_stage_2.tsv", delimiter="\t")
 PRETRAINED_MODEL_NAME = 'bert-large-uncased'
bert_path = ".. /input/bert-base-uncased/"
tokenizer = BertTokenizer.from_pretrained(bert_path)
 pad_len = 300<data_type_conversions>
y=train['SalePrice']
x=train.drop(['SalePrice'],axis=1)
 X_train, X_valid, y_train, y_valid=train_test_split(x,y,random_state=73 )
def conver_lower(df):
df['Text'] = df.apply(lambda row: row['Text'].lower() , axis = 1)
 df['A'] = df.apply(lambda row: row['A'].lower() , axis = 1)
 df['B'] = df.apply(lambda row: row['B'].lower() , axis = 1)
 df['Pronoun'] = df.apply(lambda row: row['Pronoun'].lower() , axis = 1)
 return df
df_train = conver_lower(df_train)
 df_test = conver_lower(df_test)
 df_val = conver_lower(df_val)
 test_2 = conver_lower(test_2 )<string_transform>
RF_f=RandomForestRegressor(bootstrap=False, max_depth=60, max_features='sqrt',
min_samples_split=4, n_estimators=1700)
 RF_f.fit(X_train,y_train)
 y_pred_RF_f=RF_f.predict(X_valid)
 print('RMSLE:', root_mean_squared_log_error(y_valid, y_pred_RF_f))
tokenizer.add_tokens(['[A]', '[B]', '[P]'])
 def insert_tag(row):
to_be_inserted = sorted([ 
(row["A-offset"], " [A] "), 
(row["B-offset"], " [B] "), 
(row["Pronoun-offset"], " [P] ")
 ], key=lambda x: x[0], reverse=True)
 text = row["Text"]
for offset, tag in to_be_inserted:
text = text[:offset] + tag + text[offset:]
return text
def tokenize(text, tokenizer):
entries = {}
final_tokens = []
for token in tokenizer.tokenize(text):
if token in("[A]", "[B]", "[P]"):
entries[token] = len(final_tokens)
 continue
final_tokens.append(token)
 return final_tokens,(entries["[A]"], entries["[B]"], entries["[P]"])
 def target(row):
if int(row['A-coref'])== 1:
return 0
elif int(row['B-coref'])== 1:
return 1
else:
return 2

 def children(m):
return m if isinstance(m,(list, tuple)) else list(m.children())
 def set_trainable_attr(m, b):
m.trainable = b
for p in m.parameters() :
p.requires_grad = b
def apply_leaf(m, f):
c = children(m)
 if isinstance(m, nn.Module):
f(m)
 if len(c)> 0:
for l in c:
apply_leaf(l, f)
 def set_trainable(l, b):
apply_leaf(l, lambda m: set_trainable_attr(m, b))<prepare_x_and_y>
XT=ExtraTreesRegressor(n_estimators=1000,random_state=73)
 XT.fit(X_train,y_train)
 y_pred_XT=XT.predict(X_valid)
 print('RMSLE:', root_mean_squared_log_error(y_valid, y_pred_XT))
class modified_dataset(torch.utils.data.Dataset):
def __init__(self, df, tokenizer):
p_text = []
offsets = []
at_mask = []
self.y_lst = df[['A-coref', 'B-coref']].apply(lambda row: target(row), axis = 1)
 for row in tqdm(range(len(df))):
tokens, offset = tokenize(insert_tag(df.iloc[row]), tokenizer)
 bla = tokenizer.encode_plus(tokens, max_length = pad_len, pad_to_max_length = True, return_token_type_ids = False)
 p_text.append(bla['input_ids'])
 at_mask.append(bla['attention_mask'])
 offsets.append(offset)
 self.p_text = torch.tensor(p_text)
 self.offsets = torch.tensor(offsets)
 self.at_mask = torch.tensor(at_mask)
 return
def __len__(self):
return len(self.p_text)
 def __getitem__(self,item):
return self.p_text[item], self.y_lst[item], self.offsets[item], self.at_mask[item]
class modified_dataset_test(torch.utils.data.Dataset):
def __init__(self, df, tokenizer):
p_text = []
offsets = []
at_mask = []
for row in tqdm(range(len(df))):
tokens, offset = tokenize(insert_tag(df.iloc[row]), tokenizer)
 bla = tokenizer.encode_plus(tokens, max_length = pad_len, pad_to_max_length = True, return_token_type_ids = False)
 p_text.append(bla['input_ids'])
 at_mask.append(bla['attention_mask'])
 offsets.append(offset)
 self.p_text = torch.tensor(p_text)
 self.offsets = torch.tensor(offsets)
 self.at_mask = torch.tensor(at_mask)
 return
def __len__(self):
return len(self.p_text)
 def __getitem__(self,item):
return self.p_text[item], self.offsets[item], self.at_mask[item]
def collate_fun(batch):
tmp_lst = list(zip(*batch))
return torch.stack(tmp_lst[0], axis = 0), torch.tensor(tmp_lst[1]), torch.stack(tmp_lst[2], axis = 0), torch.stack(tmp_lst[3], axis = 0)
 def collate_fun2(batch):
tmp_lst = list(zip(*batch))
return torch.stack(tmp_lst[0], axis = 0), torch.stack(tmp_lst[1], axis = 0), torch.stack(tmp_lst[2], axis = 0)
 train_loader = DataLoader(
 modified_dataset(df_train, tokenizer),
batch_size=18,
collate_fn=collate_fun,
shuffle=True,
drop_last=True,
num_workers=2)
 val_loader = DataLoader(
 modified_dataset(df_val, tokenizer),
batch_size=30,
collate_fn=collate_fun,
shuffle=False,
num_workers=2)
 test_loader = DataLoader(
 modified_dataset(df_test, tokenizer),
batch_size=30,
collate_fn=collate_fun,
shuffle=False,
num_workers=2)
 test_2_loader = DataLoader(
 modified_dataset_test(test_2, tokenizer),
batch_size=30,
collate_fn=collate_fun2,
shuffle=False,
num_workers=2 )<load_pretrained>
Ada_f=AdaBoostRegressor(learning_rate=0.03, loss='exponential', n_estimators=2300,
random_state=73)
 Ada_f.fit(X_train,y_train)
 y_pred_Ada_f=Ada_f.predict(X_valid)
 print('RMSLE:', root_mean_squared_log_error(y_valid, y_pred_Ada_f))
GPU = torch.cuda.is_available()
torch.save({'train_loader':train_loader,
'test_loader':test_loader,
'val_loader':val_loader}, 'dataloader_new.pth')
 torch.save({'test_dataloader':test_2_loader},'test_loader.pth')
<load_pretrained>
GB_f=GradientBoostingRegressor(max_depth=10, max_features='sqrt',
min_samples_split=12, n_estimators=1500)
 GB_f.fit(X_train,y_train)
 y_pred_GB_f=GB_f.predict(X_valid)
 print('RMSLE:', root_mean_squared_log_error(y_valid, y_pred_GB_f))
class bert(nn.Module):
def __init__(self, bert_path):
super().__init__()
BERT = BertModel.from_pretrained(bert_path, config = BertConfig.from_pretrained(bert_path, output_hidden_states = True))
self.BERT = BERT
self.fc = nn.Sequential(nn.BatchNorm1d(self.BERT.config.hidden_size * 3),
nn.Dropout(0.4),
nn.Linear(self.BERT.config.hidden_size * 3, 600),
nn.BatchNorm1d(600),
nn.Dropout(0.4),
nn.Linear(600, 600),
nn.BatchNorm1d(600),
nn.Dropout(0.4),
nn.Linear(600,3))
def forward(self, token, at_mask, offsets, layer):
out = self.BERT(token, attention_mask = at_mask)[2][layer]
out_lst = []
for j in range(out.shape[0]):
out_lst.append(torch.stack([torch.tensor(out[j,offsets[j,0]]),torch.tensor(out[j,offsets[j,1]]),torch.tensor(out[j,offsets[j,2]])] , dim = 0))
out_lst = torch.stack([word_embedding for word_embedding in out_lst], dim = 0)
 out = out_lst.reshape(out_lst.shape[0], -1)
 out = self.fc(out)
 return out
def create_model(df_len,epoch_len):
model = bert(bert_path)
 criteria = nn.CrossEntropyLoss()
optimizer = AdamW(model.parameters() , eps = 1e-06, lr = 1e-4)
 scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=0, num_training_steps=df_len*epoch_len)
 return model, criteria, optimizer, scheduler<train_model>
HGB_f=HistGradientBoostingRegressor(learning_rate=0.02,
loss='least_absolute_deviation', max_depth=40,
max_iter=750, min_samples_leaf=2,
random_state=73)
 HGB_f.fit(X_train,y_train)
 y_pred_HGB_f=HGB_f.predict(X_valid)
 print('RMSLE:', root_mean_squared_log_error(y_valid, y_pred_HGB_f))
epoch_len = 20
model, criteria, optimizer, scheduler = create_model(len(df_train), epoch_len)
 set_trainable(model.BERT, False)
 aaa = 0
for t in range(epoch_len):
tot_loss = 0
correct_train = 0
val_loss = 0
val_correct = 0
model = model.train()
if GPU:
model = model.cuda()
for item in tqdm(train_loader):
token = item[0]
at_mask = item[3]
offsets = item[2]
target = item[1]
if GPU:
token = token.cuda()
at_mask = at_mask.cuda()
target = target.cuda()
offsets = offsets.cuda()
output = model(token, at_mask, offsets, -2)
 loss = criteria(output, target)
 tot_loss += loss.item()
correct_train += torch.sum(torch.max(torch.nn.functional.softmax(output, dim = 1), dim = 1)[1] == target)
 optimizer.zero_grad()
loss.backward()
nn.utils.clip_grad_norm_(model.parameters() , max_norm=1.0)
 optimizer.step()
scheduler.step()
with torch.no_grad() :
model = model.eval()
if GPU:
model = model.cuda()
for item in tqdm(val_loader):
token = item[0]
at_mask = item[3]
offsets = item[2]
target = item[1]
if GPU:
token = token.cuda()
at_mask = at_mask.cuda()
offsets = offsets.cuda()
target = target.cuda()
output = model(token, at_mask, offsets, -2)
 val_correct += torch.sum(torch.max(torch.nn.functional.softmax(output, dim = 1), dim = 1)[1] == target)
 if val_correct > aaa:
bst_model = model
aaa = val_correct
print(tot_loss, correct_train," ", val_correct," out of ", len(val_loader)*30 )<compute_test_metric>
XGB=XGBRegressor(n_estimators=1200,learning_rate=0.05,random_state=73)
 XGB.fit(X_train,y_train)
 y_pred_XGB=XGB.predict(X_valid)
 print('RMSLE:', root_mean_squared_log_error(y_valid, y_pred_XGB))
def predict(df, dataloader, model):
tmp_array = np.zeros(( len(df), 3))
with torch.no_grad() :
model = model.eval()
if GPU:
model = model.cuda()
j = 0
for item in tqdm(dataloader):
token = item[0]
at_mask = item[2]
offsets = item[1]
if GPU:
token = token.cuda()
at_mask = at_mask.cuda()
offsets = offsets.cuda()
output = model(token, at_mask, offsets, -2)
 for zz in output.cpu() :
tmp_array[j] = zz
j+=1
return tmp_array<predict_on_test>
a = predict(test_2, test_2_loader, bst_model )<save_to_csv>
XGB_f=XGBRegressor(base_score=0.5, booster='gbtree', colsample_bylevel=1,
colsample_bynode=1, colsample_bytree=0.6, gamma=0.5, gpu_id=-1,
importance_type='gain', interaction_constraints='',
learning_rate=0.05, max_delta_step=0, max_depth=4,
min_child_weight=1, monotone_constraints='() ',
n_estimators=1200, n_jobs=0, num_parallel_tree=1, random_state=73,
reg_alpha=0, reg_lambda=1, scale_pos_weight=1, subsample=1.0,
tree_method='exact', validate_parameters=1, verbosity=None)
 XGB_f.fit(X_train,y_train)
 y_pred_XGB_f=XGB_f.predict(X_valid)
 print('RMSLE:', root_mean_squared_log_error(y_valid, y_pred_XGB_f))
bla = test_2[['ID']].merge(pd.DataFrame(torch.nn.functional.softmax(torch.tensor(a), dim = 1 ).numpy()), left_index=True, right_index=True ).set_index('ID')
 bla.columns = ['A', 'B', 'NEITHER']
bla.to_csv('sbmsn2.csv' )<save_model>
XGB_model_f=XGBRegressor(base_score=0.5, booster='gbtree', colsample_bylevel=1,
colsample_bynode=1, colsample_bytree=0.5, gamma=0.2, gpu_id=-1,
importance_type='gain', interaction_constraints='',
learning_rate=0.05, max_delta_step=0, max_depth=4,
min_child_weight=2, monotone_constraints='() ',
n_estimators=1000, n_jobs=0, num_parallel_tree=1, random_state=73,
reg_alpha=0, reg_lambda=1, scale_pos_weight=1, subsample=1.0,
tree_method='exact', validate_parameters=1, verbosity=None)
 XGB_model_f.fit(X_train,y_train)
 y_pred_XGB_model_f=XGB_model_f.predict(X_valid)
 print('RMSLE:', root_mean_squared_log_error(y_valid, y_pred_XGB_model_f))
torch.save({'model':bst_model}, 'model1.pth' )<load_pretrained>
LGBg_f=LGBMRegressor(objective='regression',num_leaves=5,
learning_rate=0.05, n_estimators=720,
max_bin = 55, bagging_fraction = 0.8,
bagging_freq = 5, feature_fraction = 0.2319,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf =6, min_sum_hessian_in_leaf = 11)
 LGBg_f.fit(X_train,y_train)
 y_pred_LGBg_f=LGBg_f.predict(X_valid)
 print('RMSLE:', root_mean_squared_log_error(y_valid, y_pred_LGBg_f))
tst_model = torch.load('/kaggle/input/gendered-model/model1.pth')['model']<define_variables>
Vote=VotingRegressor([('gb',GB_f),('hgb',HGB_f),('xgb',XGB_f),('lgb',LGBg_f)])
 Vote.fit(X_train,y_train)
 y_pred_Vote=Vote.predict(X_valid)
 print('RMSLE:', root_mean_squared_log_error(y_valid, y_pred_Vote))
DATA_ROOT = '.. /input/'
GAP_DATA_FOLDER = os.path.join(DATA_ROOT, 'gap-coreference')
 SUB_DATA_FOLDER = os.path.join(DATA_ROOT, 'gendered-pronoun-resolution')
 FAST_TEXT_DATA_FOLDER = os.path.join(DATA_ROOT, 'fasttext-crawl-300d-2m' )<load_from_csv>
estimators=[('gb',GB_f),('hgb',HGB_f),('xgb',XGB_f),('lgb',LGBg_f)]
Stack=StackingRegressor(estimators=estimators,final_estimator=RandomForestRegressor(n_estimators=800,random_state=42))
Stack.fit(X_train,y_train)
 y_pred_Stack=Stack.predict(X_valid)
 print('RMSLE:', root_mean_squared_log_error(y_valid, y_pred_Stack))
sub_test_df_path = os.path.join(SUB_DATA_FOLDER, 'test_stage_2.tsv')
 test_df_path = os.path.join(GAP_DATA_FOLDER, 'gap-test.tsv')
 train_df_path = os.path.join(GAP_DATA_FOLDER, 'gap-development.tsv')
 dev_df_path = os.path.join(GAP_DATA_FOLDER, 'gap-validation.tsv')
 train_df = pd.read_csv(train_df_path, sep='\t')
 test_df = pd.read_csv(test_df_path, sep='\t')
 dev_df = pd.read_csv(dev_df_path, sep='\t')
 sub_test_df = pd.read_csv(sub_test_df_path, sep='\t')
<choose_model_class>
Voting=VotingRegressor([('gb',GB_f),('hgb',HGB_f),('xgb',XGB_f),('lgb',LGBg_f)])
 Voting.fit(x,y )
spacy_model = "en_core_web_lg"<import_modules>
preds = Voting.predict(test)
 output = pd.DataFrame({'Id': test_raw.Id,
'SalePrice': preds})
 output.to_csv('submission.csv', index=False )