Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<predict_on_test>
pip install shap
et_preds = et.predict(test.drop(et_drop_cols, axis=1))
et_probs = et.predict_proba(test.drop(et_drop_cols, axis=1)) * 2
model_count = 2<create_dataframe>
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
cv_submission = pd.DataFrame()
cv_submission['Id'] = X_cv.Id
cv_submission['Target_et'] = et_cv_preds<create_dataframe>
data_train = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv')
 data_test = pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/test.csv")
 data_test = data_test.drop(['Id'],1)
 data = data_train.append(data_test)
 data = data.drop(['SalePrice'],1)
 data = data.drop(['Id'],1)
submission = pd.DataFrame()
submission['Id'] = test.Id
submission['Target_et'] = et_preds<save_to_csv>
print("p-value for shapiro-wilk test is ",shapiro(target)[1] )
submission[["Id", "Target_et"]].to_csv("20180725_etc_1.csv", header=["Id", "Target"], index=False )<compute_test_metric>
corr_cols = ['OverallQual', 'GrLivArea', 'GarageCars', 'TotalBsmtSF',
'FullBath', 'YearBuilt']
print("Correlations")
 for i in corr_cols:
print(i,"and target",np.corrcoef(target,data_train[i])[0][1])
ranked_features = feature_importance(et, X_tr.drop(et_drop_cols, axis=1))<define_variables>
nas_ratio = data.isnull().sum() / data.shape[0] * 100
nas_ratio = nas_ratio.drop(nas_ratio[nas_ratio==0].index ).sort_values(ascending=False)
 nas_ratio
extra_lgb_drop_cols = ['techo_LE',
'abastagua_LE',
'hogar_total',
'v18q',
'mobilephone',
'hacdor',
'tamhog',
'hacapo',
'parentesco1'
]<create_dataframe>
data["PoolQC"] = data["PoolQC"].fillna("None")
 data["MiscFeature"] = data["MiscFeature"].fillna("None")
 data["Alley"] = data["Alley"].fillna("None")
 data["Fence"] = data["Fence"].fillna("None")
 data["FireplaceQu"] = data["FireplaceQu"].fillna("None")
 for col in('GarageType', 'GarageFinish', 'GarageQual', 'GarageCond'):
data[col] = data[col].fillna('None')
 for col in('GarageYrBlt', 'GarageArea', 'GarageCars'):
data[col] = data[col].fillna(0)
 for col in('BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF','TotalBsmtSF', 'BsmtFullBath', 'BsmtHalfBath'):
data[col] = data[col].fillna(0)
 for col in('BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2'):
data[col] = data[col].fillna('None')
 data["MasVnrType"] = data["MasVnrType"].fillna("None")
 data["MasVnrArea"] = data["MasVnrArea"].fillna(0)
 data["Functional"] = data["Functional"].fillna("Typ")
 data['Electrical'] = data['Electrical'].fillna(data['Electrical'].mode() [0])
 data['KitchenQual'] = data['KitchenQual'].fillna(data['KitchenQual'].mode() [0])
 data['Exterior1st'] = data['Exterior1st'].fillna(data['Exterior1st'].mode() [0])
 data['Exterior2nd'] = data['Exterior2nd'].fillna(data['Exterior2nd'].mode() [0])
 data['SaleType'] = data['SaleType'].fillna(data['SaleType'].mode() [0])
 data['MSSubClass'] = data['MSSubClass'].fillna("None")
 data["LotFrontage"] = data.groupby("Neighborhood")["LotFrontage"].transform(
 lambda x: x.fillna(x.median()))
data['MSZoning'] = data['MSZoning'].fillna(data['MSZoning'].mode() [0])
 data = data.drop(['Utilities'], axis=1)
 data = data.drop(['Street'], axis=1)
 data = data.drop(['Condition2'], axis=1)
 data = data.drop(['RoofMatl'], axis=1)
 data = data.drop(['PoolArea'], axis=1)
 data = data.drop(['3SsnPorch'], axis=1)
 data = data.drop(['LandSlope'], axis=1)
 data = data.drop(['LowQualFinSF'], axis=1)
 data = data.drop(['Electrical'], axis=1)
 data = data.drop(['MiscFeature'], axis=1)
 label_enc = LabelEncoder()
data["YearBuilt"] = label_enc.fit_transform(data["YearBuilt"])
 data["YearRemodAdd"] = label_enc.fit_transform(data["YearRemodAdd"])
 data["GarageYrBlt"] = label_enc.fit_transform(data["GarageYrBlt"])
 data['MoSold'] = label_enc.fit_transform(data['MoSold'])
 data['OverallCond'] = label_enc.fit_transform(data['OverallCond'])
 data['MSSubClass'] = label_enc.fit_transform(data['MSSubClass'])
 data['YrSold'] = label_enc.fit_transform(data['YrSold'])
 data['TotalSF'] = data['TotalBsmtSF'] + data['1stFlrSF'] + data['2ndFlrSF']
print("Now there are",data.isnull().sum().sum() ,"NA values now" )
lgb_drop_cols = ['Id', 'idhogar'] + extra_lgb_drop_cols
train_data = lgb.Dataset(X_tr.drop(lgb_drop_cols, axis=1), y_tr-1)
 max_depth = 15
params = {
'boosting_type': 'gbdt',
'objective': 'multiclass',
'metric': 'multi_logloss',
'max_depth': max_depth,
'num_leaves': 2**max_depth-1,
'learning_rate': 0.05,
'feature_fraction': 0.75,
'bagging_fraction': 0.85,
'bagging_freq': 5,
'verbose': -1,
'num_threads': 4,
'lambda_l2': 0.85,
'min_gain_to_split': 0,
'num_class': len(np.unique(y_tr)) ,
}
model = lgb.train(params, train_data, num_boost_round=500 )<compute_train_metric>
print(data.skew().sort_values(ascending=False ).head(10))
data["LotArea"], _ = stats.boxcox(data["LotArea"].values)
 print("skew after:",data["LotArea"].skew() )
lgb_cv_probs = model.predict(X_cv.drop(lgb_drop_cols, axis=1))
lgb_cv_preds = np.argmax(lgb_cv_probs, axis=1)+ 1
print("Accuracy:", accuracy_score(y_cv, lgb_cv_preds))
print("F1:", f1_score(y_cv, lgb_cv_preds, average="micro"))<compute_test_metric>
to_std = ['LotFrontage', 'LotArea',
'YearBuilt','YearRemodAdd',
'MasVnrArea','BsmtFinSF1','BsmtFinSF2','BsmtUnfSF','TotalBsmtSF',
'1stFlrSF','BsmtFullBath','GarageYrBlt','GarageArea',
'WoodDeckSF','MoSold','YrSold','TotalSF']
for i in range(len(to_std)) :
data[to_std[i]] = data[to_std[i]].astype("float64" )
print("Accuracy:", accuracy_score(y_cv, lgb_cv_preds))
print("F1:", f1_score(y_cv, lgb_cv_preds, average="micro"))<predict_on_test>
for c in to_std:
std = StandardScaler()
std.fit(list(np.array(data[c] ).reshape(-1,1)))
 data[c] = std.transform(list(np.array(data[c] ).reshape(-1,1)) )
lgb_probs = model.predict(test.drop(lgb_drop_cols, axis=1))
lgb_preds = np.argmax(lgb_probs, axis=1)+ 1<define_variables>
data = pd.get_dummies(data )
cv_probs = et_cv_probs + lgb_cv_probs + lgb_cv_probs + lgb_cv_probs
test_probs = et_probs + lgb_probs + lgb_probs + lgb_probs
model_count += 3<save_to_csv>
X = data[:1460].values
y = data_train.SalePrice.values
submission['Target_lgb'] = lgb_preds
submission['Target_lgb4'] = lgb_preds
submission[["Id", "Target_lgb"]].to_csv("20180725_lgb_1.csv", header=["Id", "Target"], index=False )<prepare_output>
test = data[1460:]
cv_submission['Target_lgb'] = lgb_cv_preds
cv_submission['Target_lgb4'] = lgb_cv_preds<define_variables>
n_folds = 5
def rmsle_cv(model):
kf = KFold(n_folds, shuffle=True ).get_n_splits(X)
 rmse= np.sqrt(-cross_val_score(model, X, y, scoring="neg_mean_squared_error", cv = kf))
return(rmse.mean())
extra_rf_drop_cols = []<define_variables>
model_xgb = xgb.XGBRegressor(colsample_bytree=0.46, gamma=0.047,
learning_rate=0.05, max_depth=3,
min_child_weight=1.785, n_estimators=2200,
reg_alpha=0.465, reg_lambda=0.86,
subsample=0.522)
 print("xgb: ",rmsle_cv(model_xgb))
extra_rf_drop_cols = ['r4t3_to_tamhog', 'min_rez', 'dependency_no', 'dependency_yes']<train_model>
model_lgb = lgb.LGBMRegressor(objective='regression',num_leaves=5,
learning_rate=0.05, n_estimators=720,
max_bin = 55, bagging_fraction = 0.82,
bagging_freq = 5, feature_fraction = 0.232,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf =6, min_sum_hessian_in_leaf = 11)
 print("lgb:",rmsle_cv(model_lgb))
rf_drop_cols = ['Id', 'idhogar', 'parentesco1'] + extra_rf_drop_cols
rf = RandomForestClassifier(n_estimators=350, max_depth=31, verbose=1, min_impurity_decrease=1e-6, n_jobs=-1)
 rf.fit(X_tr.drop(rf_drop_cols, axis=1), y_tr )<compute_train_metric>
cat_boost = CatBoostRegressor(learning_rate=0.05,
n_estimators=800,depth=3,silent=True)
 print("catboost: ",rmsle_cv(cat_boost))
rf_cv_preds = rf.predict(X_cv.drop(rf_drop_cols, axis=1))
rf_cv_probs = rf.predict_proba(X_cv.drop(rf_drop_cols, axis=1))
print("Accuracy:", accuracy_score(y_cv, rf_cv_preds))
print("F1:", f1_score(y_cv, rf_cv_preds, average="micro"))<compute_test_metric>
Enet = ElasticNet(alpha=0.005,
l1_ratio=0.85,max_iter=15000)
 print("enet: ", rmsle_cv(Enet))
print("Accuracy:", accuracy_score(y_cv, rf_cv_preds))
print("F1:", f1_score(y_cv, rf_cv_preds, average="micro"))<predict_on_test>
class AveragingModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, models):
self.models = models
def fit(self, X, y):
self.models_ = [clone(x)for x in self.models]
for model in self.models_:
model.fit(X, y)
 return self
def predict(self, X):
predictions = np.column_stack([
model.predict(X)for model in self.models_
])
 return np.mean(predictions, axis=1 )
rf_preds = rf.predict(test.drop(rf_drop_cols, axis=1))
rf_probs = rf.predict_proba(test.drop(rf_drop_cols, axis=1))<define_variables>
models_forAverage = AveragingModels(models=(cat_boost,model_xgb,model_lgb))
print(rmsle_cv(models_forAverage))
cv_probs = cv_probs + rf_cv_probs
test_probs = test_probs + rf_probs
model_count += 1<save_to_csv>
cat_bst = CatBoostRegressor(iterations=1000,learning_rate=0.1,silent=True)
 cat_bst.fit(X,y )
submission['Target_rf'] = rf_preds
submission[["Id", "Target_rf"]].to_csv("20180725_rf_1.csv", header=["Id", "Target"], index=False )<prepare_output>
feature_interactions = cat_bst.get_feature_importance(catboost.Pool(X,y),type="Interaction" )
cv_submission['Target_rf'] = rf_cv_preds<compute_test_metric>
fi_new = []
for k, item in enumerate(feature_interactions):
first = data.dtypes.index[feature_interactions[k][0]]
second = data.dtypes.index[feature_interactions[k][1]]
if first != second:
fi_new.append([first+ " " + second, feature_interactions[k][2]] )
rf_features = feature_importance(rf, X_tr.drop(rf_drop_cols, axis=1))<categorify>
interaction_score = pd.DataFrame(fi_new,columns=['Feature-Pair','Interaction Score'])
 interaction_score = interaction_score.sort_values(by="Interaction Score",ascending=False,
inplace=False,na_position='last' )
def dprint(*args, **kwargs):
print("[{}] ".format(datetime.datetime.now().strftime("%Y-%m-%d %H:%M")) + \
" ".join(map(str,args)) , **kwargs)
 id_name = 'Id'
target_name = 'Target'
df_all = pd.concat([train, test], axis=0)
 cols = [f_ for f_ in df_all.columns if df_all[f_].dtype == 'object' and f_ != id_name]
for c in cols:
le = LabelEncoder()
le.fit(df_all[c].astype(str))
train[c] = le.transform(train[c].astype(str))
test[c] = le.transform(test[c].astype(str))
gc.collect()
print("Done.")
 cnt = 0
p_buf = []
n_splits = 5
n_repeats = 1
kf = RepeatedKFold(
 n_splits=n_splits,
n_repeats=n_repeats,
random_state=0)
 err_buf = []
cols_to_drop = [
id_name,
target_name,
'idhogar',
]
X = train2.drop(cols_to_drop, axis=1, errors='ignore')
 feature_names = list(X.columns)
 X = X.fillna(0)
 X = X.values
y = target
classes = np.unique(y)
 dprint('Number of classes: {}'.format(len(classes)))
 c2i = {}
i2c = {}
for i, c in enumerate(classes):
c2i[c] = i
i2c[i] = c
y_le = np.array([c2i[c] for c in y])
 X_test = test.drop(cols_to_drop, axis=1, errors='ignore')
 X_test = X_test.fillna(0)
 X_test = X_test.values
id_test = test[id_name].values
dprint(X.shape, y.shape)
 dprint(X_test.shape)
 n_features = X.shape[1]
max_depth = 12
lgb_params = {
'boosting_type': 'gbdt',
'objective': 'multiclass',
'metric': 'multi_logloss',
'max_depth': max_depth,
'num_leaves':(2**max_depth)-1,
'learning_rate': 0.05,
'feature_fraction': 0.75,
'bagging_fraction': 0.75,
'bagging_freq': 5,
'verbose': -1,
'num_threads': 4,
'lambda_l2': 0.85,
'min_gain_to_split': 0,
'num_class': len(np.unique(y)) ,
}
for i in range(5):
cv_hhs = np.random.choice(households, size=int(len(households)* 0.25), replace=False)
 valid_index = np.isin(train_hhs, cv_hhs)
 train_index = ~np.isin(train_hhs, cv_hhs)
 print('Fold {}/{}*{}'.format(cnt + 1, n_splits, n_repeats))
params = lgb_params.copy()
sampler = ADASYN(random_state=0)
 X_train, y_train = sampler.fit_sample(X[train_index], y_le[train_index])
 lgb_train = lgb.Dataset(
 X_train,
y_train,
feature_name=feature_names, 
)
 lgb_train.raw_data = None
lgb_valid = lgb.Dataset(
 X[valid_index],
y_le[valid_index],
feature_name=feature_names, 
)
 lgb_valid.raw_data = None
model = lgb.train(
 params,
lgb_train,
num_boost_round=99999,
valid_sets=[lgb_train, lgb_valid],
early_stopping_rounds=200,
verbose_eval=100, 
)
 if cnt == 0:
importance = model.feature_importance()
model_fnames = model.feature_name()
tuples = sorted(zip(model_fnames, importance), key=lambda x: x[1])[::-1]
tuples = [x for x in tuples if x[1] > 0]
print('Important features:')
 for i in range(10):
if i < len(tuples):
print(i, tuples[i])
 else:
break
del importance, model_fnames, tuples
p = model.predict(X[valid_index], num_iteration=model.best_iteration)
 err = f1_score(y_le[valid_index], np.argmax(p, axis=1), average='micro')
 dprint('{} F1: {}'.format(cnt + 1, err))
p = model.predict(X_test, num_iteration=model.best_iteration)
 if len(p_buf)== 0:
p_buf = np.array(p, dtype=np.float16)
 else:
p_buf += np.array(p, dtype=np.float16)
 err_buf.append(err)
 cnt += 1
del model, lgb_train, lgb_valid, p
gc.collect()
err_mean = np.mean(err_buf)
 err_std = np.std(err_buf)
 print('F1 = {:.6f} +/- {:.6f}'.format(err_mean, err_std))
preds_lgb2 = p_buf/cnt
subm = pd.DataFrame()
subm[id_name] = id_test
for i in range(preds_lgb2.shape[1]):
subm[i2c[i]] = preds_lgb2[:, i]
submission["Target_lgb2"] = [i2c[np.argmax(p)] for p in preds_lgb2]
submission[["Id", "Target_lgb2"]].to_csv("20180725_lgb2_preds.csv", header=["Id", "Target"], index=False )<define_variables>
extended_catboost = CatBoostRegressor(per_float_feature_quantization=['3:border_count=1024', '33:border_count=1024'],
depth=8,
learning_rate=0.025,
n_estimators=2000,silent=True)
 print(rmsle_cv(extended_catboost))
test_probs = test_probs + preds_lgb2
model_count += 1<init_hyperparams>
models_forAverage2 = AveragingModels(models=(extended_catboost,model_xgb))
print(rmsle_cv(models_forAverage2))
lgb_drop_cols = ['Id', 'idhogar']
opt_parameters = {'colsample_bytree': 0.93, 'min_child_samples': 56, 'num_leaves': 19, 'subsample': 0.84}<compute_train_metric>
data_test = pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/test.csv")
 IDS = data_test['Id']
def evaluate_macroF1_lgb(truth, predictions):
pred_labels = predictions.reshape(len(np.unique(truth)) ,-1 ).argmax(axis=0)
 f1 = f1_score(truth, pred_labels, average='macro')
 return('macroF1', f1, True)
 fit_params={"early_stopping_rounds":300,
"eval_metric" : evaluate_macroF1_lgb,
"eval_set" : [(X_tr.drop(lgb_drop_cols, axis=1),y_tr-1),(X_cv.drop(lgb_drop_cols, axis=1),y_cv-1)],
'eval_names': ['train', 'valid'],
'verbose': False,
'categorical_feature': 'auto'}
def learning_rate_power_0997(current_iter):
base_learning_rate = 0.1
min_learning_rate = 0.02
lr = base_learning_rate * np.power (.995, current_iter)
 return max(lr, min_learning_rate)
 fit_params['verbose'] = 200
fit_params['callbacks'] = [lgb.reset_parameter(learning_rate=learning_rate_power_0997)]<train_model>
model_xgb2 = xgb.XGBRegressor(colsample_bytree=0.46, gamma=0.047,
learning_rate=0.05, max_depth=3,
min_child_weight=1.785, n_estimators=2200,
reg_alpha=0.465, reg_lambda=0.86,
subsample=0.522)
 model_xgb2.fit(data[:1460],y)
 extended_catboost2 = CatBoostRegressor(per_float_feature_quantization=['3:border_count=1024', '33:border_count=1024'],
depth=8,
learning_rate=0.025,
n_estimators=2000,silent=True)
 extended_catboost2.fit(data[:1460],y)
def _parallel_fit_estimator(estimator, X, y, sample_weight=None, **fit_params):

 if sample_weight is not None:
estimator.fit(X, y, sample_weight=sample_weight, **fit_params)
 else:
estimator.fit(X, y, **fit_params)
 return estimator
class VotingClassifierLGBM(VotingClassifier):

 def fit(self, X, y, sample_weight=None, **fit_params):
if isinstance(y, np.ndarray)and len(y.shape)> 1 and y.shape[1] > 1:
raise NotImplementedError('Multilabel and multi-output'
' classification is not supported.')
 if self.voting not in('soft', 'hard'):
raise ValueError("Voting must be 'soft' or 'hard'; got(voting=%r)"
% self.voting)
 if self.estimators is None or len(self.estimators)== 0:
raise AttributeError('Invalid `estimators` attribute, `estimators`'
' should be a list of(string, estimator)'
' tuples')
 if(self.weights is not None and
len(self.weights)!= len(self.estimators)) :
raise ValueError('Number of classifiers and weights must be equal'
'; got %d weights, %d estimators'
%(len(self.weights), len(self.estimators)))
 if sample_weight is not None:
for name, step in self.estimators:
if not has_fit_parameter(step, 'sample_weight'):
raise ValueError('Underlying estimator '%s' does not'
' support sample weights.' % name)
 names, clfs = zip(*self.estimators)
 self._validate_names(names)
 n_isnone = np.sum([clf is None for _, clf in self.estimators])
 if n_isnone == len(self.estimators):
raise ValueError('All estimators are None.At least one is '
'required to be a classifier!')
 self.le_ = LabelEncoder().fit(y)
 self.classes_ = self.le_.classes_
self.estimators_ = []
transformed_y = self.le_.transform(y)
 self.estimators_ = Parallel(n_jobs=self.n_jobs )(
 delayed(_parallel_fit_estimator )(clone(clf), X, transformed_y,
sample_weight=sample_weight, **fit_params)
 for clf in clfs if clf is not None)
 return self<train_model>
sub = pd.DataFrame()
sub['Id'] = IDS
sub['SalePrice'] =(model_xgb2.predict(data[1460:])*0.5)+(extended_catboost2.predict(data[1460:])*0.5)
 sub.to_csv('submission.csv',index=False )