Kaggle competition: Costa Rican Household Poverty Level Prediction
<SOS> metric: MacroFScore Kaggle data source: costa-rican-household-poverty-level-prediction<categorify>
warnings.filterwarnings('ignore' )
cols =('FireplaceQu', 'BsmtQual', 'BsmtCond', 'GarageQual', 'GarageCond',
'ExterQual', 'ExterCond','HeatingQC', 'PoolQC', 'KitchenQual', 'BsmtFinType1',
'BsmtFinType2', 'Functional', 'Fence', 'BsmtExposure', 'GarageFinish', 'LandSlope',
'LotShape', 'PavedDrive', 'Street', 'Alley', 'CentralAir', 'MSSubClass', 'OverallCond',
'YrSold', 'MoSold')
 for c in cols:
lbl = LabelEncoder()
lbl.fit(list(data[c].values))
data[c] = lbl.transform(list(data[c].values))
print('Shape data: {}'.format(data.shape))<categorify>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 test['Target'] = np.nan
data = train.append(test, sort = True )
data = pd.get_dummies(data)
 print(data.shape )<prepare_x_and_y>
train_valid = train.loc[train['parentesco1'] == 1, ['idhogar', 'Id', 'Target']].copy()
test_valid = test.loc[test['parentesco1'] == 1, ['idhogar', 'Id']].copy()
submission_base = test[['Id', 'idhogar']]
train = data[:1460]
test = data[1460:].drop('SalePrice', axis = 1 )<prepare_x_and_y>
mapping = {"yes": 1, "no": 0}
data['dependency'] = data['dependency'].replace(mapping ).astype(np.float64)
 data['edjefa'] = data['edjefa'].replace(mapping ).astype(np.float64)
 data['edjefe'] = data['edjefe'].replace(mapping ).astype(np.float64)
 data[['dependency', 'edjefa', 'edjefe']].describe()
X = train.drop('SalePrice', axis = 1)
 y = train['SalePrice']<split>
data['v18q1'] = data['v18q1'].fillna(0)
 data.loc[(data['tipovivi1'] == 1), 'v2a1'] = 0
data['v2a1-missing'] = data['v2a1'].isnull()
data.loc[(( data['age'] > 19)|(data['age'] < 7)) &(data['rez_esc'].isnull()), 'rez_esc'] = 0
data['rez_esc-missing'] = data['rez_esc'].isnull()
data.loc[data['rez_esc'] > 5, 'rez_esc'] = 5
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state = 42 )<choose_model_class>
data['hhsize-diff'] = data['tamviv'] - data['hhsize']
elec = []
for i, row in data.iterrows() :
if row['noelec'] == 1:
elec.append(0)
 elif row['coopele'] == 1:
elec.append(1)
 elif row['public'] == 1:
elec.append(2)
 elif row['planpri'] == 1:
elec.append(3)
 else:
elec.append(np.nan)
 data['elec'] = elec
data['elec-missing'] = data['elec'].isnull()
data['walls'] = np.argmax(np.array(data[['epared1', 'epared2', 'epared3']]),
axis = 1)
 data['roof'] = np.argmax(np.array(data[['etecho1', 'etecho2', 'etecho3']]),
axis = 1)
 data['floor'] = np.argmax(np.array(data[['eviv1', 'eviv2', 'eviv3']]),
axis = 1)
 data['walls+roof+floor'] = data['walls'] + data['roof'] + data['floor']
data['warning'] = 1 *(data['sanitario1'] + 
(data['elec'] == 0)+
data['pisonotiene'] +
data['abastaguano'] + 
(data['cielorazo'] == 0))
data['bonus'] = 1 *(data['refrig'] +
data['computer'] + 
(data['v18q1'] > 0)+
data['television'])
 data['phones-per-capita'] = data['qmobilephone'] / data['tamviv']
data['tablets-per-capita'] = data['v18q1'] / data['tamviv']
data['rooms-per-capita'] = data['rooms'] / data['tamviv']
data['rent-per-capita'] = data['v2a1'] / data['tamviv']
data['inst'] = np.argmax(np.array(data[[c for c in data if c.startswith('instl')]]), axis = 1)
 data['escolari/age'] = data['escolari'] / data['age']
data['inst/age'] = data['inst'] / data['age']
data['tech'] = data['v18q'] + data['mobilephone']
print('Data shape: ', data.shape )
xgb = XGBRegressor(booster='gbtree', colsample_bylevel=1,
colsample_bynode=1,colsample_bytree=0.6,
gamma=0, importance_type='gain',
learning_rate=0.01, max_delta_step=0,
max_depth=4,min_child_weight=1.5,
n_estimators=2500, n_jobs=1, nthread=None,
objective='reg:linear', reg_alpha=0.4640,
reg_lambda=0.6, scale_pos_weight=1,
silent=None, subsample=0.8, verbosity=1 )<choose_model_class>
data = data[[x for x in data if not x.startswith('SQB')]]
data = data.drop(columns = ['agesq'])
 data.shape
lgbm = LGBMRegressor(objective='regression',
num_leaves=4,
learning_rate=0.01,
n_estimators=11000,
max_bin=200,
bagging_fraction=0.75,
bagging_freq=5,
bagging_seed=7,
feature_fraction=0.4 )<choose_model_class>
data = data.drop(columns = to_drop )
gboost = GradientBoostingRegressor(n_estimators=3000,
learning_rate=0.05,
max_depth=4,
max_features='sqrt',
min_samples_leaf=15,
min_samples_split=10,
loss='huber',
random_state =5 )<train_model>
import featuretools.variable_types as vtypes
xgb.fit(X_train, y_train)
 lgbm.fit(X_train, y_train, eval_metric='rmsle')
 gboost.fit(X_train, y_train )<predict_on_test>
hh_bool = ['hacdor', 'hacapo', 'v14a', 'refrig', 'paredblolad', 'paredzocalo',
'paredpreb','pisocemento', 'pareddes', 'paredmad',
'paredzinc', 'paredfibras', 'paredother', 'pisomoscer', 'pisoother',
'pisonatur', 'pisonotiene', 'pisomadera',
'techozinc', 'techoentrepiso', 'techocane', 'techootro', 'cielorazo',
'abastaguadentro', 'abastaguafuera', 'abastaguano',
'public', 'planpri', 'noelec', 'coopele', 'sanitario1',
'sanitario2', 'sanitario3', 'sanitario5', 'sanitario6',
'energcocinar1', 'energcocinar2', 'energcocinar3', 'energcocinar4',
'elimbasu1', 'elimbasu2', 'elimbasu3', 'elimbasu4',
'elimbasu5', 'elimbasu6', 'epared1', 'epared2', 'epared3',
'etecho1', 'etecho2', 'etecho3', 'eviv1', 'eviv2', 'eviv3',
'tipovivi1', 'tipovivi2', 'tipovivi3', 'tipovivi4', 'tipovivi5',
'computer', 'television', 'lugar1', 'lugar2', 'lugar3',
'lugar4', 'lugar5', 'lugar6', 'area1', 'area2', 'v2a1-missing', 'elec-missing']
hh_ordered = [ 'rooms', 'r4h1', 'r4h2', 'r4h3', 'r4m1','r4m2','r4m3', 'r4t1', 'r4t2',
'r4t3', 'v18q1', 'tamhog','tamviv','hhsize','hogar_nin','hhsize-diff',
'elec', 'walls', 'roof', 'floor', 'walls+roof+floor', 'warning', 'bonus',
'hogar_adul','hogar_mayor','hogar_total', 'bedrooms', 'qmobilephone']
hh_cont = ['v2a1', 'dependency', 'edjefe', 'edjefa', 'meaneduc', 'overcrowding',
'phones-per-capita', 'tablets-per-capita', 'rooms-per-capita', 'rent-per-capita']
pred2 = xgb.predict(X_test)
 pred3 = lgbm.predict(X_test)
 pred4 = gboost.predict(X_test )<compute_test_metric>
ind_bool = ['v18q', 'dis', 'male', 'female', 'estadocivil1', 'estadocivil2', 'estadocivil3',
'estadocivil4', 'estadocivil5', 'estadocivil6', 'estadocivil7',
'parentesco1', 'parentesco2', 'parentesco3', 'parentesco4', 'parentesco5',
'parentesco6', 'parentesco7', 'parentesco8', 'parentesco9', 'parentesco10',
'parentesco11', 'parentesco12', 'instlevel1', 'instlevel2', 'instlevel3',
'instlevel4', 'instlevel5', 'instlevel6', 'instlevel7', 'instlevel8',
'instlevel9', 'mobilephone', 'rez_esc-missing']
ind_ordered = ['age', 'escolari', 'rez_esc', 'inst', 'tech']
ind_cont = ['escolari/age', 'inst/age']
print('Erro médio logarítmico de raiz quadrado test(XGB)= ' + str(math.sqrt(metrics.mean_squared_log_error(y_test, pred2))))
print('Erro médio logarítmico de raiz quadrado test(LGBM)= ' + str(math.sqrt(metrics.mean_squared_log_error(y_test, pred3))))
print('Erro médio logarítmico de raiz quadrado test(GBoost)= ' + str(math.sqrt(metrics.mean_squared_log_error(y_test, pred4))))<train_model>
to_remove = []
for l in [hh_ordered, hh_bool, hh_cont, ind_bool, ind_ordered, ind_cont]:
for c in l:
if c not in data:
to_remove.append(c )
lgbm.fit(X, y)
 xgb.fit(X ,y)
 gboost.fit(X, y )<predict_on_test>
for l in [hh_ordered, hh_bool, hh_cont, ind_bool, ind_ordered, ind_cont]:
for c in to_remove:
if c in l:
l.remove(c )
prediction_lgbm = np.expm1(lgbm.predict(test))
prediction_xgb = np.expm1(xgb.predict(test))
prediction_gboost = np.expm1(gboost.predict(test))<define_variables>
for variable in(hh_bool + ind_bool):
data[variable] = data[variable].astype('bool' )
<prepare_output>
for variable in(hh_cont + ind_cont):
data[variable] = data[variable].astype(float )
prediction =(prediction_lgbm * 0.38 + prediction_gboost * 0.36 + prediction_xgb * 0.26 )<create_dataframe>
for variable in(hh_ordered + ind_ordered):
try:
data[variable] = data[variable].astype(int)
 except Exception as e:
print(f'Could not convert {variable} because of missing values.' )
submission = pd.DataFrame({"Id": test.index,"SalePrice": prediction} )<save_to_csv>
%matplotlib inline
submission.to_csv('submission.csv', index=False )<set_options>
es = ft.EntitySet(id = 'households')
 es.entity_from_dataframe(entity_id = 'data',
dataframe = data,
index = 'Id' )
pd.options.mode.chained_assignment = None
%matplotlib inline<load_from_csv>
es.normalize_entity(base_entity_id='data',
new_entity_id='household',
index = 'idhogar',
additional_variables = hh_bool + hh_ordered + hh_cont + ['Target'])
 es
test = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/test.csv")
 train = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/train.csv" )<split>
feature_matrix, feature_names = ft.dfs(entityset=es,
target_entity = 'household',
max_depth = 2,
verbose = 1,
n_jobs = -1,
chunk_size = 100)
X_train, X_test, y_train, y_test = train_test_split(train, train["SalePrice"], test_size=0.3, random_state=42)
 X_train.shape , X_test.shape<count_missing_values>
drop_cols = []
for col in feature_matrix:
if col == 'Target':
pass
else:
if 'Target' in col:
drop_cols.append(col)
 print(drop_cols)
 feature_matrix = feature_matrix[[x for x in feature_matrix if x not in drop_cols]]
feature_matrix.head()
cat_var_na = []
for label, content in X_train.items() :
if pd.api.types.is_string_dtype(content)and train[label].isnull().sum() *100/ len(X_train)>0:
cat_var_na.append(label)
 X_train[cat_var_na].isnull().sum() *100/len(X_train )<count_missing_values>
feature_matrix = feature_matrix[[x for x in feature_matrix if x not in to_drop]]
cat_var_40 = []
cat_var_10 = []
for var in cat_var_na:
if X_train[var].isnull().sum() *100/len(X_train)>=5:
cat_var_40.append(var)
 else:
cat_var_10.append(var )<categorify>
train = feature_matrix[feature_matrix['Target'].notnull() ].reset_index()
test = feature_matrix[feature_matrix['Target'].isnull() ].reset_index()
train.head()
X_train[cat_var_40] = X_train[cat_var_40].fillna("Missing")
 X_test[cat_var_40] = X_test[cat_var_40].fillna("Missing")
 test[cat_var_40] = test[cat_var_40].fillna("Missing" )<categorify>
corrs = train.corr()
corrs['Target'].sort_values(ascending = True ).head(10 )
for var in cat_var_10:
mode_val = X_train[var].mode() [0]
X_train[var] = X_train[var].fillna(mode_val)
 X_test[var] = X_test[var].fillna(mode_val)
 test[var] = test[var].fillna(mode_val )<define_variables>
corrs['Target'].sort_values(ascending = True ).dropna().tail(10 )
cat_var = []
for var in X_train.columns:
if X_train[var].dtype == "O":
cat_var.append(var )<count_values>
train = train[train['idhogar'].isin(list(train_valid['idhogar'])) ]
train.head()
def find_frequent(df, var, rare):

 df=df.copy()
temp = df[var].value_counts() /len(df)
 return temp[temp>rare].index 
<feature_engineering>
test = test[test['idhogar'].isin(list(test_valid['idhogar'])) ]
test.head()
for var in cat_var:
frequent_ls = find_frequent(X_train, var, 0.03)
 X_train[var] = np.where(X_train[var].isin(frequent_ls), X_train[var], "Rare")
 X_test[var] = np.where(X_test[var].isin(frequent_ls), X_test[var], "Rare")
 test[var] = np.where(test[var].isin(frequent_ls), test[var], "Rare" )<categorify>
train_labels = np.array(train.pop('Target')).reshape(( -1,))
test_ids = list(test.pop('idhogar'))
def encode(train, test, c_test, target):
ordered_label = train.groupby([var])[target].mean().sort_values().index
ordinal_label = {k: i for i,k in enumerate(ordered_label,0)}
train[var] = train[var].map(ordinal_label)
 test[var] = test[var].map(ordinal_label)
 c_test[var] = c_test[var].map(ordinal_label)
<categorify>
train, test = train.align(test, axis = 1, join = 'inner')
 all_features = list(train.columns)
 train.shape
for var in cat_var:
encode(X_train, X_test, test, "SalePrice" )<count_missing_values>
%%capture
num_var_na = []
for label, content in X_train.items() :
if pd.api.types.is_numeric_dtype(content)and X_train[label].isnull().sum() >0:
num_var_na.append(label)
 X_train[num_var_na].isnull().sum() *100/len(X_train)
<data_type_conversions>
def macro_f1_score(labels, predictions):
predictions = predictions.reshape(len(np.unique(labels)) , -1 ).argmax(axis = 0)
 metric_value = f1_score(labels, predictions, average = 'macro')
 return 'macro_f1', metric_value, True
for var in num_var_na:
var_mode = X_train[var].mode() [0]
X_train[var + "_na"] = np.where(X_train[var].isnull() ,1,0)
 X_test[var + "_na"] = np.where(X_test[var].isnull() ,1,0)
 test[var + "_na"] = np.where(test[var].isnull() ,1,0)
 X_train[var] = X_train[var].fillna(var_mode)
 X_test[var] = X_test[var].fillna(var_mode)
 test[var] = test[var].fillna(var_mode)
 X_train[num_var_na].isnull().sum() *100/len(X_train )<define_variables>
from IPython.display import display
cont_var = ["LotFrontage", "LotArea", "1stFlrSF", "GrLivArea"] 
<categorify>
def model_gbm(features, labels, test_features, test_ids,
nfolds = 5, return_preds = False, hyp = None):

 feature_names = list(features.columns)
 hyp_OPTaaS = { 'boosting_type': 'dart',
'colsample_bytree': 0.9843467236959204,
'learning_rate': 0.11598629586769524,
'min_child_samples': 44,
'num_leaves': 49,
'reg_alpha': 0.35397370408131534,
'reg_lambda': 0.5904910774606467,
'subsample': 0.6299872254632797,
'subsample_for_bin': 60611}
model = lgb.LGBMClassifier(**hyp_OPTaaS, class_weight = 'balanced',
objective = 'multiclass', n_jobs = -1, n_estimators = 10000)
 strkfold = StratifiedKFold(n_splits = nfolds, shuffle = True)
 predictions = pd.DataFrame()
importances = np.zeros(len(feature_names))
features = np.array(features)
 test_features = np.array(test_features)
 labels = np.array(labels ).reshape(( -1))
valid_scores = []
for i,(train_indices, valid_indices)in enumerate(strkfold.split(features, labels)) :
fold_predictions = pd.DataFrame()
X_train = features[train_indices]
X_valid = features[valid_indices]
y_train = labels[train_indices]
y_valid = labels[valid_indices]
model.fit(X_train, y_train, early_stopping_rounds = 100,
eval_metric = macro_f1_score,
eval_set = [(X_train, y_train),(X_valid, y_valid)],
eval_names = ['train', 'valid'],
verbose = 200)
 valid_scores.append(model.best_score_['valid']['macro_f1'])
 fold_probabilitites = model.predict_proba(test_features)
 for j in range(4):
fold_predictions[(j + 1)] = fold_probabilitites[:, j]
fold_predictions['idhogar'] = test_ids
fold_predictions['fold'] =(i+1)
 predictions = predictions.append(fold_predictions)
 importances += model.feature_importances_ / nfolds
display(f'Fold {i + 1}, Validation Score: {round(valid_scores[i], 5)}, Estimators Trained: {model.best_iteration_}')
 feature_importances = pd.DataFrame({'feature': feature_names,
'importance': importances})
 valid_scores = np.array(valid_scores)
 display(f'{nfolds} cross validation score: {round(valid_scores.mean() , 5)} with std: {round(valid_scores.std() , 5)}.')
 if return_preds:
predictions['Target'] = predictions[[1, 2, 3, 4]].idxmax(axis = 1)
 predictions['confidence'] = predictions[[1, 2, 3, 4]].max(axis = 1)
 return predictions, feature_importances
predictions = predictions.groupby('idhogar', as_index = False ).mean()
predictions['Target'] = predictions[[1, 2, 3, 4]].idxmax(axis = 1)
 predictions['confidence'] = predictions[[1, 2, 3, 4]].max(axis = 1)
 predictions = predictions.drop(columns = ['fold'])
 submission = submission_base.merge(predictions[['idhogar', 'Target']], on = 'idhogar', how = 'left' ).drop(columns = ['idhogar'])
 submission['Target'] = submission['Target'].fillna(4 ).astype(np.int8)
 return submission, feature_importances, valid_scores
def quantile_boundary(df, var, lower_bound, upper_bound):

 lower = df[var].quantile(lower_bound)
 upper = df[var].quantile(upper_bound)
 return lower, upper<feature_engineering>
%%capture --no-display
submission, feature_importances, valid_scores = model_gbm(train, train_labels, test, test_ids, 5)
 results = pd.DataFrame({'version': ['default_5fold'],
'F1-mean': [valid_scores.mean() ],
'F1-std': [valid_scores.std() ]} )
lower,upper = quantile_boundary(X_train, "LotFrontage",0.065,0.95)
 X_train["LotFrontage"] = np.where(X_train["LotFrontage"]<lower,lower,
np.where(X_train["LotFrontage"]>upper,upper,X_train["LotFrontage"]))
lower,upper = quantile_boundary(X_train, "LotArea",0.0065,0.955)
 X_train["LotArea"] = np.where(X_train["LotArea"]<lower,lower,
np.where(X_train["LotArea"]>upper,upper,X_train["LotArea"]))
lower,upper = quantile_boundary(X_train, "1stFlrSF",0.0065,0.985)
 X_train["1stFlrSF"] = np.where(X_train["1stFlrSF"]<lower,lower,
np.where(X_train["1stFlrSF"]>upper,upper,X_train["1stFlrSF"]))
lower,upper = quantile_boundary(X_train, "GrLivArea",0.0065,0.98)
 X_train["GrLivArea"] = np.where(X_train["GrLivArea"]<lower,lower,
np.where(X_train["GrLivArea"]>upper,upper,X_train["GrLivArea"])) 
<feature_engineering>
submission.to_csv('ft_baseline.csv', index = False )
X_train["SalePrice"]=np.log(X_train["SalePrice"] )<categorify>
def range_calc(numeric):
return np.max(numeric)- np.min(numeric)
 range_ = make_agg_primitive(function = range_calc,
input_types = [ft.variable_types.Numeric],
return_type = ft.variable_types.Numeric )
def year_to_age(df, var):

 df[var] = df["YrSold"]-df[var]
return df<categorify>
def p_corr_calc(numeric1, numeric2):
return np.corrcoef(numeric1, numeric2)[0, 1]
pcorr_ = make_agg_primitive(function = p_corr_calc,
input_types = [ft.variable_types.Numeric, ft.variable_types.Numeric],
return_type = ft.variable_types.Numeric )
year_var = ['YearBuilt', 'YearRemodAdd', 'GarageYrBlt']
for var in year_var:
X_train = year_to_age(X_train, var)
 X_test = year_to_age(X_test, var)
 test = year_to_age(test, var)
<feature_engineering>
def s_corr_calc(numeric1, numeric2):
return spearmanr(numeric1, numeric2)[0]
scorr_ = make_agg_primitive(function = s_corr_calc,
input_types = [ft.variable_types.Numeric, ft.variable_types.Numeric],
return_type = ft.variable_types.Numeric )
lower,upper = quantile_boundary(X_train, "YearBuilt",0.065,0.97)
 X_train["YearBuilt"] = np.where(X_train["YearBuilt"]<lower,lower,
np.where(X_train["YearBuilt"]>upper,upper,X_train["YearBuilt"])) 
<define_variables>
%%capture
feature_matrix_add, feature_names_add = ft.dfs(entityset=es, target_entity = 'household',
agg_primitives = ['min', 'max', 'mean', 'percent_true', 'all', 'any',
'sum', 'skew', 'std', range_],
trans_primitives = [], drop_exact = all_features,
max_depth = 2,
verbose = 1, n_jobs = -1,
chunk_size = 100 )
feature_scale = [var for var in X_train.columns if var not in ['Id', 'SalePrice']]
len(feature_scale )<categorify>
all_features += [str(x.get_name())for x in feature_names_add]
feature_matrix = pd.concat([feature_matrix, feature_matrix_add], axis = 1)
 feature_matrix.shape
for var in feature_scale:
mode_val = X_train[var].mode() [0]
test[var] = test[var].fillna(mode_val )<normalization>
def post_process(feature_matrix,
missing_threshold = 0.95,
correlation_threshold = 0.95):
start_features = feature_matrix.shape[1]
feature_matrix = feature_matrix.iloc[:, ~feature_matrix.columns.duplicated() ]
n_duplicated = start_features - feature_matrix.shape[1]
print(f'There were {n_duplicated} duplicated features.')
 feature_matrix = feature_matrix.replace({np.inf: np.nan, -np.inf:np.nan} ).reset_index()
ids = list(feature_matrix.pop('idhogar'))
labels = list(feature_matrix.pop('Target'))
drop_cols = []
for col in feature_matrix:
if col == 'Target':
pass
else:
if 'Target' in col:
drop_cols.append(col)
 feature_matrix = feature_matrix[[x for x in feature_matrix if x not in drop_cols]]
feature_matrix = pd.get_dummies(feature_matrix)
 n_features_start = feature_matrix.shape[1]
print('Original shape: ', feature_matrix.shape)
 missing = pd.DataFrame(feature_matrix.isnull().sum())
 missing['fraction'] = missing[0] / feature_matrix.shape[0]
missing.sort_values('fraction', ascending = False, inplace = True)
 missing_cols = list(missing[missing['fraction'] > missing_threshold].index)
 n_missing_cols = len(missing_cols)
 feature_matrix = feature_matrix[[x for x in feature_matrix if x not in missing_cols]]
print('{} missing columns with threshold: {}.'.format(n_missing_cols, missing_threshold))
unique_counts = pd.DataFrame(feature_matrix.nunique() ).sort_values(0, ascending = True)
 zero_variance_cols = list(unique_counts[unique_counts[0] == 1].index)
 n_zero_variance_cols = len(zero_variance_cols)
 feature_matrix = feature_matrix[[x for x in feature_matrix if x not in zero_variance_cols]]
print('{} zero variance columns.'.format(n_zero_variance_cols))
corr_matrix = feature_matrix.corr()
upper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k = 1 ).astype(np.bool))
to_drop = [column for column in upper.columns if any(upper[column].abs() > correlation_threshold)]
n_collinear = len(to_drop)
 feature_matrix = feature_matrix[[x for x in feature_matrix if x not in to_drop]]
print('{} collinear columns removed with correlation above {}.'.format(n_collinear, correlation_threshold))
total_removed = n_duplicated, n_missing_cols + n_zero_variance_cols + n_collinear
print('Total columns removed: ', total_removed)
 print('Shape after feature selection: {}.'.format(feature_matrix.shape))
feature_matrix['idhogar'] = ids
feature_matrix['Target'] = labels
train = feature_matrix[feature_matrix['Target'].notnull() ]
test = feature_matrix[feature_matrix['Target'].isnull() ]
train = train[train['idhogar'].isin(list(train_valid['idhogar'])) ]
test = test[test['idhogar'].isin(list(test_valid['idhogar'])) ]
train_labels = np.array(train.pop('Target')).reshape(( -1,))
test_ids = list(test.pop('idhogar'))
train, test = train.align(test, join = 'inner', axis = 1)
 assert(len(train_labels)== train.shape[0]), "Labels must be same length as number of training observations"
assert(len(test_ids)== test.shape[0]), "Must be equal number of test ids as testing observations"
return train, train_labels, test, test_ids
scaler=MinMaxScaler()
X_train[feature_scale] = scaler.fit_transform(X_train[feature_scale])
 X_test[feature_scale] = scaler.transform(X_test[feature_scale])
 test[feature_scale] = scaler.transform(test[feature_scale] )<prepare_x_and_y>
train, train_labels, test, test_ids = post_process(feature_matrix )
y_train = X_train['SalePrice'].values
y_test = X_test['SalePrice'].values
X_train.drop(['Id', 'SalePrice'], axis=1, inplace=True)
 X_test.drop(['Id', 'SalePrice'], axis=1, inplace=True )<features_selection>
%%capture
feature_matrix_add, feature_names_add = ft.dfs(entityset=es, target_entity = 'household',
agg_primitives = ['min', 'max', 'mean', 'percent_true', 'all', 'any',
'sum', 'skew', 'std', range_, pcorr_],
trans_primitives = [], drop_contains = list(all_features),
max_depth = 2, max_features = 1000,
verbose = 1, n_jobs = -1,
chunk_size = 100 )
sfm = SelectFromModel(Lasso(alpha = 0.001, random_state = 0))
sfm.fit(X_train,y_train)
 feature_selected = X_train.columns[(sfm.get_support())]
len(feature_selected )<drop_column>
all_features += [str(x.get_name())for x in feature_names_add]
feature_matrix = pd.concat([feature_matrix, feature_matrix_add], axis = 1)
 train, train_labels, test, test_ids = post_process(feature_matrix)
 train.shape
X_train=X_train[feature_selected]
X_test = X_test[feature_selected]
test = test[feature_selected]<compute_train_metric>
%%capture --no-display
submission, feature_importances, valid_scores = model_gbm(train, train_labels, test, test_ids, 5)
 results = results.append(pd.DataFrame({'version': ['additionalft_5fold'],
'F1-mean': [valid_scores.mean() ],
'F1-std': [valid_scores.std() ]}))
def rmse_cv(model):
rmse= np.sqrt(-cross_val_score(model, X_train, y_train, scoring="neg_mean_squared_error", cv = 14))
return(rmse )<find_best_params>
%%capture
feature_matrix_add, feature_names_add = ft.dfs(entityset=es, target_entity = 'household',
agg_primitives = ['min', 'max', 'mean', 'percent_true', 'all', 'any',
'sum', 'skew', 'std', range_, pcorr_],
trans_primitives = ['divide'], drop_contains = list(all_features),
max_depth = 2, max_features = 1000,
verbose = 1, n_jobs = -1,
chunk_size = 1000 )
model_ridge = Ridge()
alphas = [0.08,0.05, 0.1,0.001, 0.3, 1, 3, 5, 10,11, 15, 30, 50, 75]
cv_ridge = [rmse_cv(Ridge(alpha = alpha)).mean()
for alpha in alphas]
cv_ridge = pd.Series(cv_ridge, index = alphas)
 cv_ridge.min()<train_on_grid>
all_features += [str(x.get_name())for x in feature_names_add]
feature_matrix = pd.concat([feature_matrix, feature_matrix_add], axis = 1)
 train, train_labels, test, test_ids = post_process(feature_matrix)
 train.shape
model_lasso = LassoCV(alphas = [0.0001,0.0009,0.001,0.002,0.003,0.01,0.1,1,10,100] ).fit(X_train, y_train)
 rmse_cv(model_lasso ).min()<train_model>
%%capture --no-display
submission, feature_importances, valid_scores = model_gbm(train, train_labels, test, test_ids, 5)
 results = results.append(pd.DataFrame({'version': ['divide1000_5fold'],
'F1-mean': [valid_scores.mean() ],
'F1-std': [valid_scores.std() ]}))
submission.to_csv('divide1000_featuretools.csv', index = False )
model_xgb = xgb.XGBRegressor(n_estimators=450, max_depth=3, learning_rate=0.06)
 model_xgb.fit(X_train, y_train )<prepare_output>
feature_matrix_add, feature_names_add = ft.dfs(entityset=es, target_entity = 'household',
agg_primitives = ['min', 'max', 'mean', 'percent_true', 'all', 'any',
'sum', 'skew', 'std', range_, pcorr_],
trans_primitives = ['divide'], drop_contains = list(all_features),
max_depth = 2, max_features = 1500,
verbose = 1, n_jobs = -1,
chunk_size = 1000 )
xgb_preds = np.expm1(model_xgb.predict(X_test))
lasso_preds = np.expm1(model_lasso.predict(X_test))
linear_preds = 0.4*xgb_preds+0.6*lasso_preds
predictions = pd.DataFrame({"xgb":xgb_preds, "lasso":lasso_preds,"actual":y_test,
"linear_preds": linear_preds })
 predictions["Xgb_error"]=predictions["actual"]-predictions["xgb"]
predictions["Lasso_error"]=predictions["actual"]-predictions["lasso"]
predictions["Linear_error"]=predictions["linear_preds"]-predictions["lasso"]<compute_test_metric>
all_features += [str(x.get_name())for x in feature_names_add]
feature_matrix = pd.concat([feature_matrix, feature_matrix_add], axis = 1)
 train, train_labels, test, test_ids = post_process(feature_matrix)
 train.shape
print(mean_squared_error(y_test,predictions["linear_preds"])**(1/2))
print(r2_score(y_test,predictions["linear_preds"]))<predict_on_test>
%%capture --no-display
submission, feature_importances, valid_scores = model_gbm(train, train_labels, test, test_ids, 5)
 results = results.append(pd.DataFrame({'version': ['divide1500_5fold'],
'F1-mean': [valid_scores.mean() ],
'F1-std': [valid_scores.std() ]}))
submission.to_csv('divide1500_featuretools.csv', index = False )
test_prediction= 0.4*np.expm1(model_xgb.predict(test)) + 0.6*np.expm1(model_lasso.predict(test))<save_to_csv>
%%capture
feature_matrix_add, feature_names_add = ft.dfs(entityset=es, target_entity = 'household',
agg_primitives = ['min', 'max', 'mean', 'percent_true', 'all', 'any',
'sum', 'skew', 'std', range_, pcorr_],
trans_primitives = ['divide'], drop_exact = list(all_features),
max_depth = 2, max_features = 2000,
verbose = 1, n_jobs = -1,
chunk_size = 100 )
test = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/test.csv")
 submission = pd.DataFrame({
"Id": test["Id"],
"SalePrice": test_prediction
})
 submission.to_csv('submission.csv', index=False )<import_modules>
all_features += [str(x.get_name())for x in feature_names_add]
feature_matrix = pd.concat([feature_matrix, feature_matrix_add], axis = 1)
 train, train_labels, test, test_ids = post_process(feature_matrix)
 train.shape
import numpy as np
import pandas as pd
from sklearn import preprocessing
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from xgboost import XGBRegressor
import sklearn.metrics as metrics
import math
import seaborn as sns
from sklearn.ensemble import RandomForestRegressor
from lightgbm import LGBMRegressor
from catboost import CatBoostRegressor<load_from_csv>
%%capture --no-display
submission, feature_importances, valid_scores = model_gbm(train, train_labels, test, test_ids, 5)
 results = results.append(pd.DataFrame({'version': ['divide2000_5fold'],
'F1-mean': [valid_scores.mean() ],
'F1-std': [valid_scores.std() ]}))
submission.to_csv('divide2000_featuretools.csv', index = False )
train_df = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 test_df = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv' )<define_variables>
%%capture --no-display
submission, feature_importances, valid_scores = model_gbm(train, train_labels, test, test_ids, 10)
 results = results.append(pd.DataFrame({'version': ['divide2000_10fold'],
'F1-mean': [valid_scores.mean() ],
'F1-std': [valid_scores.std() ]}))
submission.to_csv('divide2000_10fold_featuretools.csv', index = False )
labels = train_df['SalePrice']
labels<drop_column>
%%capture
feature_matrix_add, feature_names_add = ft.dfs(entityset=es, target_entity = 'household',
agg_primitives = ['min', 'max', 'mean', 'percent_true', 'all', 'any',
'sum', 'skew', 'std', range_, pcorr_],
trans_primitives = ['divide'], drop_contains = list(all_features),
max_depth = 2, max_features = 5000,
verbose = 1, n_jobs = -1,
chunk_size = 100 )
train_df.drop(['SalePrice'], axis=1, inplace=True )<define_variables>
all_features += [str(x.get_name())for x in feature_names_add]
feature_matrix = pd.concat([feature_matrix, feature_matrix_add], axis = 1)
 train, train_labels, test, test_ids = post_process(feature_matrix)
 train.shape
train_idx = train_df.shape[0]
train_idx<concatenate>
%%capture --no-display
submission, feature_importances, valid_scores = model_gbm(train, train_labels, test, test_ids, 5)
 results = results.append(pd.DataFrame({'version': ['divide5000_5fold'], 'F1-mean': [valid_scores.mean() ], 'F1-std': [valid_scores.std() ]}))
submission.to_csv('divide5000_featuretools.csv', index = False )
new_df = pd.concat([train_df, test_df], ignore_index=True, axis=0 )<feature_engineering>
%%capture --no-display
submission, feature_importances, valid_scores = model_gbm(train, train_labels, test, test_ids, 10)
 results = results.append(pd.DataFrame({'version': ['divide5000_10fold'],
'F1-mean': [valid_scores.mean() ],
'F1-std': [valid_scores.std() ]}))
submission.to_csv('divide5000_10fold_featuretools.csv', index = False )
nan_pct = [new_df[col].isna().sum() * 100/new_df.shape[0] for col in new_df.columns]
nan_df = pd.DataFrame(columns=['column', 'percentage'])
 nan_df['column'] = new_df.columns
nan_df['percentage'] = nan_pct
nan_df[nan_df['percentage'] > 50]<drop_column>
feature_matrix = feature_matrix.iloc[:, ~feature_matrix.columns.duplicated() ].reset_index()
new_df.drop(nan_df[nan_df['percentage'] > 50]['column'].unique() , axis=1, inplace=True )<filter>
train_ids = list(feature_matrix[(feature_matrix['Target'].notnull())&(feature_matrix['idhogar'].isin(list(train_valid['idhogar'])))]['idhogar'] )
nan_df[(nan_df['percentage'] < 50)&(nan_df['percentage'] > 5)]<drop_column>
train['Target'] = train_labels
test['Target'] = np.nan
train['idhogar'] = train_ids
test['idhogar'] = test_ids
data = train.append(test)
 results.to_csv('model_results.csv', index = True)
 data.to_csv('ft_5000_important.csv', index = False )
<data_type_conversions><EOS> 
print('Final shape of data(with testing joined to training): ', data.shape )