Kaggle competition: House Prices - Advanced Regression Techniques
test_leak, test_result = compiled_leak_result_test(max_nlags=38 )<create_dataframe>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
test_result = pd.DataFrame.from_dict(test_result, orient='columns')
 test_result<save_to_csv>
ntrain = train.shape[0]
ntest = test.shape[0]
y_train = train.SalePrice.values
all_data = pd.concat(( train, test)).reset_index(drop=True)
 all_data.drop(['SalePrice'], axis=1, inplace=True)
 print("all_data size is : {}".format(all_data.shape))
test_result.to_csv('test_leaky_stat.csv', index=False )<categorify>
all_data_na =(all_data.isnull().sum() / len(all_data)) * 100
print(len(all_data_na))
all_data_na = all_data_na.drop(all_data_na[all_data_na == 0].index ).sort_values(ascending=False)[:30]
print(len(all_data_na))
missing_data = pd.DataFrame({'Missing Ratio' :all_data_na})
 missing_data.head(20 )
test_leak = rewrite_compiled_leak(test_leak, best_lag)
 test_leak[['ID']+leaky_cols+["compiled_leak", "nonzero_mean"]].head()<categorify>
all_data["PoolQC"] = all_data["PoolQC"].fillna("None" )
NLAGS = 15
all_df = get_all_leak(all_df, cols=cols, nlags=NLAGS )<merge>
all_data["MiscFeature"] = all_data["MiscFeature"].fillna("None" )
leaky_cols = ["leaked_target_"+str(i)for i in range(NLAGS)]
train = train.join(all_df.set_index("ID")[leaky_cols], on="ID", how="left")
 test = test.join(all_df.set_index("ID")[leaky_cols], on="ID", how="left" )<feature_engineering>
all_data["Alley"] = all_data["Alley"].fillna("None" )
train["nonzero_mean"] = train[transact_cols].apply(lambda x: np.expm1(np.log1p(x[x!=0] ).mean()), axis=1)
 test["nonzero_mean"] = test[transact_cols].apply(lambda x: np.expm1(np.log1p(x[x!=0] ).mean()), axis=1 )<feature_engineering>
all_data["Fence"] = all_data["Fence"].fillna("None" )
train["compiled_leak"] = 0
test["compiled_leak"] = 0
for i in range(NLAGS):
train.loc[train["compiled_leak"] == 0, "compiled_leak"] = train.loc[train["compiled_leak"] == 0, "leaked_target_"+str(i)]
test.loc[test["compiled_leak"] == 0, "compiled_leak"] = test.loc[test["compiled_leak"] == 0, "leaked_target_"+str(i)]
print("Leak values found in train and test ", sum(train["compiled_leak"] > 0), sum(test["compiled_leak"] > 0))
print("% of correct leaks values in train ", sum(train["compiled_leak"] == train["target"])/sum(train["compiled_leak"] > 0))
train.loc[train["compiled_leak"] == 0, "compiled_leak"] = train.loc[train["compiled_leak"] == 0, "nonzero_mean"]
test.loc[test["compiled_leak"] == 0, "compiled_leak"] = test.loc[test["compiled_leak"] == 0, "nonzero_mean"]<compute_test_metric>
all_data["FireplaceQu"] = all_data["FireplaceQu"].fillna("None" )
np.sqrt(mean_squared_error(y, np.log1p(train["compiled_leak"] ).fillna(14.49)) )<save_to_csv>
all_data["LotFrontage"] = all_data.groupby("Neighborhood")["LotFrontage"].transform(
 lambda x: x.fillna(x.median()))
sub = test[["ID"]]
sub["target"] = test["compiled_leak"]
sub.to_csv("baseline_submission_with_leaks.csv", index=False )<import_modules>
for col in('GarageType', 'GarageFinish', 'GarageQual', 'GarageCond'):
all_data[col] = all_data[col].fillna('None' )
from sklearn.datasets import load_boston
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_absolute_error
from catboost import CatBoostRegressor
from skopt import gp_minimize, forest_minimize
from skopt.space import Real, Categorical, Integer
from skopt.plots import plot_convergence
from skopt.plots import plot_objective, plot_evaluations
from skopt.utils import use_named_args
from sklearn.preprocessing import MinMaxScaler, StandardScaler
import matplotlib.pyplot as plt
from xgboost import XGBRegressor
from vecstack import stacking
import pandas as pd
import numpy as np
from sklearn.metrics import mean_squared_error
import os<load_from_csv>
for col in('GarageYrBlt', 'GarageArea', 'GarageCars'):
all_data[col] = all_data[col].fillna(0 )
temp = pd.read_csv('.. /input/santander-value-prediction-challenge/test.csv')
 test_ID = temp['ID'].values
del temp<load_from_csv>
for col in('BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF','TotalBsmtSF', 'BsmtFullBath', 'BsmtHalfBath'):
all_data[col] = all_data[col].fillna(0 )
train = pd.read_csv('.. /input/46-features-lgb/46feat_trainData.csv.gz',compression='gzip')
 test = pd.read_csv('.. /input/46-features-lgb/46feat_testData.csv.gz',compression='gzip' )<split>
for col in('BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2'):
all_data[col] = all_data[col].fillna('None' )
X_train, X_test = train_test_split(train, test_size=0.2, random_state=5 )<prepare_x_and_y>
all_data["MasVnrType"] = all_data["MasVnrType"].fillna("None")
 all_data["MasVnrArea"] = all_data["MasVnrArea"].fillna(0 )
y_train = X_train['log_target']
y_test = X_test['log_target']
t2 = 'log_target'
foo = train.columns
foo = list(foo)
 foo.remove(t2)
 colNames = foo<choose_model_class>
all_data['MSZoning'] = all_data['MSZoning'].fillna(all_data['MSZoning'].mode() [0] )
models = [
CatBoostRegressor(iterations=200,
learning_rate=0.03,
depth=4,
loss_function='RMSE',
eval_metric='RMSE',
random_seed=99,
od_type='Iter',
od_wait=50,
logging_level='Silent'),
CatBoostRegressor(iterations=500,
learning_rate=0.06,
depth=3,
loss_function='RMSE',
eval_metric='RMSE',
random_seed=99,
od_type='Iter',
od_wait=50,
logging_level='Silent'),
XGBRegressor(eta=0.1,reg_lambda=1,reg_alpha=10),
XGBRegressor(eta=0.02,reg_lambda=1,reg_alpha=10,n_estimators=300),
XGBRegressor(eta=0.002,max_depth=15,n_estimators=200),
]<concatenate>
all_data = all_data.drop(['Utilities'], axis=1 )
S_train, S_test = stacking(models, X_train[colNames], y_train, X_test[colNames],
regression = True, metric = mean_absolute_error, n_folds = 5,
shuffle = True,random_state = 0, verbose = 2 )<init_hyperparams>
all_data["Functional"] = all_data["Functional"].fillna("Typ" )
dim_learning_rate = Real(low=1e-6, high=1e-1, prior='log-uniform',name='learning_rate')
 dim_estimators = Integer(low=50, high=500,name='n_estimators')
 dim_max_depth = Integer(low=1, high=6,name='max_depth' )<define_variables>
all_data['Electrical'] = all_data['Electrical'].fillna(all_data['Electrical'].mode() [0] )
dimensions = [dim_learning_rate,
dim_estimators,
dim_max_depth]<define_search_space>
all_data['KitchenQual'] = all_data['KitchenQual'].fillna(all_data['KitchenQual'].mode() [0] )
default_parameters = [1e-2,300,4]<predict_on_test>
all_data['Exterior1st'] = all_data['Exterior1st'].fillna(all_data['Exterior1st'].mode() [0])
 all_data['Exterior2nd'] = all_data['Exterior2nd'].fillna(all_data['Exterior2nd'].mode() [0] )
def createModel(learning_rate,n_estimators,max_depth):
model = XGBRegressor(n_estimators=n_estimators,
learning_rate=learning_rate,
max_depth=max_depth,
random_state=0)
 model = model.fit(S_train, y_train)
 y_pred = model.predict(S_test)
 lv = mean_absolute_error(y_test, y_pred)
 return lv 
<train_model>
all_data['SaleType'] = all_data['SaleType'].fillna(all_data['SaleType'].mode() [0] )
@use_named_args(dimensions=dimensions)
 def fitness(learning_rate,n_estimators,max_depth):

 print('learning rate: {0:.1e}'.format(learning_rate))
print('estimators:', n_estimators)
 print('max depth:', max_depth)
 lv= createModel(learning_rate=learning_rate,
n_estimators=n_estimators,
max_depth = max_depth)
 print()
print("Error: {}".format(lv))
print()
return lv<compute_test_metric>
all_data['MSSubClass'] = all_data['MSSubClass'].fillna("None")
error = fitness(default_parameters )<train_on_grid>
all_data_na =(all_data.isnull().sum() / len(all_data)) * 100
all_data_na = all_data_na.drop(all_data_na[all_data_na == 0].index ).sort_values(ascending=False)
 missing_data = pd.DataFrame({'Missing Ratio' :all_data_na})
 missing_data.head()
search_result = gp_minimize(func=fitness,
dimensions=dimensions,
acq_func='EI',
n_calls=20,
x0=default_parameters )<train_on_grid>
all_data['MSSubClass'] = all_data['MSSubClass'].apply(str)
 all_data['OverallCond'] = all_data['OverallCond'].astype(str)
 all_data['YrSold'] = all_data['YrSold'].astype(str)
 all_data['MoSold'] = all_data['MoSold'].astype(str)
search_result.x<train_model>
cols =('FireplaceQu', 'BsmtQual', 'BsmtCond', 'GarageQual', 'GarageCond',
'ExterQual', 'ExterCond','HeatingQC', 'PoolQC', 'KitchenQual', 'BsmtFinType1',
'BsmtFinType2', 'Functional', 'Fence', 'BsmtExposure', 'GarageFinish', 'LandSlope',
'LotShape', 'PavedDrive', 'Street', 'Alley', 'CentralAir', 'MSSubClass', 'OverallCond',
'YrSold', 'MoSold')
 for c in cols:
lbl = LabelEncoder()
lbl.fit(list(all_data[c].values))
all_data[c] = lbl.transform(list(all_data[c].values))
print('Shape all_data: {}'.format(all_data.shape))
model = XGBRegressor(n_estimators=search_result.x[1],
learning_rate=search_result.x[0],
max_depth=search_result.x[2],
random_state=0)
 model = model.fit(S_train, y_train )<predict_on_test>
all_data['TotalSF'] = all_data['TotalBsmtSF'] + all_data['1stFlrSF'] + all_data['2ndFlrSF']
y_pred = model.predict(S_test )<save_to_csv>
skewness = skewness[abs(skewness)> 0.75]
print("There are {} skewed numerical features to Box Cox transform".format(skewness.shape[0]))
skewed_features = skewness.index
lam = 0.15
for feat in skewed_features:
all_data[feat] = boxcox1p(all_data[feat], lam)
result = pd.DataFrame({'ID':test_ID
,'target':np.expm1(y_pred)})
 result.to_csv('46feat_stacked_ensemble_regr_models.csv', index=False )<set_options>
all_data = pd.get_dummies(all_data)
 print(all_data.shape )
tqdm.pandas()
sns.set(style="white", color_codes=True)
 %matplotlib inline<load_from_csv>
train = all_data[:ntrain]
test = all_data[ntrain:]
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 train.head()<categorify>
from sklearn.linear_model import ElasticNet, Lasso, BayesianRidge, LassoLarsIC
from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor
from sklearn.kernel_ridge import KernelRidge
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import RobustScaler
from sklearn.base import BaseEstimator, TransformerMixin, RegressorMixin, clone
from sklearn.model_selection import KFold, cross_val_score, train_test_split
from sklearn.metrics import mean_squared_error
import xgboost as xgb
import lightgbm as lgb
def aggregate_row(row):
non_zero_values = row.iloc[row.nonzero() ].astype(float)
 if non_zero_values.empty:
aggs = {
'non_zero_mean': np.nan,
'non_zero_std': np.nan,
'non_zero_max': np.nan,
'non_zero_min': np.nan,
'non_zero_sum': np.nan,
'non_zero_skewness': np.nan,
'non_zero_kurtosis': np.nan,
'non_zero_median': np.nan,
'non_zero_q1': np.nan,
'non_zero_q3': np.nan,
'non_zero_gmean': np.nan,
'non_zero_log_mean': np.nan,
'non_zero_log_std': np.nan,
'non_zero_log_max': np.nan,
'non_zero_log_min': np.nan,
'non_zero_log_sum': np.nan,
'non_zero_log_skewness': np.nan,
'non_zero_log_kurtosis': np.nan,
'non_zero_log_median': np.nan,
'non_zero_log_q1': np.nan,
'non_zero_log_q3': np.nan,
'non_zero_log_gmean': np.nan,
'non_zero_count': np.nan,
'non_zero_fraction': np.nan
}
else:
aggs = {
'non_zero_mean': non_zero_values.mean() ,
'non_zero_std': non_zero_values.std() ,
'non_zero_max': non_zero_values.max() ,
'non_zero_min': non_zero_values.min() ,
'non_zero_sum': non_zero_values.sum() ,
'non_zero_skewness': skew(non_zero_values),
'non_zero_kurtosis': kurtosis(non_zero_values),
'non_zero_median': non_zero_values.median() ,
'non_zero_q1': np.percentile(non_zero_values, q=25),
'non_zero_q3': np.percentile(non_zero_values, q=75),
'non_zero_gmean': gmean(non_zero_values),
'non_zero_log_mean': np.log1p(non_zero_values ).mean() ,
'non_zero_log_std': np.log1p(non_zero_values ).std() ,
'non_zero_log_max': np.log1p(non_zero_values ).max() ,
'non_zero_log_min': np.log1p(non_zero_values ).min() ,
'non_zero_log_sum': np.log1p(non_zero_values ).sum() ,
'non_zero_log_skewness': skew(np.log1p(non_zero_values)) ,
'non_zero_log_kurtosis': kurtosis(np.log1p(non_zero_values)) ,
'non_zero_log_median': np.log1p(non_zero_values ).median() ,
'non_zero_log_q1': np.percentile(np.log1p(non_zero_values), q=25),
'non_zero_log_q3': np.percentile(np.log1p(non_zero_values), q=75),
'non_zero_log_gmean': gmean(np.log1p(non_zero_values)) ,
'non_zero_count': non_zero_values.count() ,
'non_zero_fraction': non_zero_values.count() / row.count()
}
return pd.Series(aggs, index=list(aggs.keys()))<split>
n_folds = 5
def rmsle_cv(model):
kf = KFold(n_folds, shuffle=True, random_state=42 ).get_n_splits(train.values)
 rmse= np.sqrt(-cross_val_score(model, train.values, y_train, scoring="neg_mean_squared_error", cv = kf))
return(rmse )
eng_features = train.iloc[:, 2:].progress_apply(aggregate_row, axis=1)
 eng_features_test = test.iloc[:, 1:].progress_apply(aggregate_row, axis=1 )<concatenate>
lasso = make_pipeline(RobustScaler() , Lasso(alpha =0.0005, random_state=1))
train_matrix = np.hstack([
eng_features.values
])
 test_matrix = np.hstack([
eng_features_test.values
] )<prepare_x_and_y>
ENet = make_pipeline(RobustScaler() , ElasticNet(alpha=0.0005, l1_ratio=.9, random_state=3))
lgb_params = {
'objective': 'binary',
'metric': 'auc',
'learning_rate': 0.1,
'num_leaves': 16,
'max_depth': -1,
'min_child_samples': 1,
'max_bin': 300,
'subsample': 1.0,
'subsample_freq': 1,
'colsample_bytree': 0.5,
'min_child_weight': 10,
'reg_lambda': 0.1,
'reg_alpha': 0.0,
'scale_pos_weight': 1,
'zero_as_missing': False,
'num_threads': -1,
}
adversarial_x = np.vstack([
train_matrix,
test_matrix
])
 adversarial_y = np.ones(len(adversarial_x))
adversarial_y[:len(train_matrix)] = 0
cv = KFold(n_splits=5, random_state=100, shuffle=True)
 train_preds = np.zeros(( len(adversarial_x)))
 for i,(train_index, valid_index)in enumerate(cv.split(adversarial_y)) :
print(f'Fold {i}')
 dtrain = lgb.Dataset(adversarial_x[train_index],
label=adversarial_y[train_index])
 dvalid = lgb.Dataset(adversarial_x[valid_index],
label=adversarial_y[valid_index])
 evals_result = {}
model = lgb.train(lgb_params, dtrain,
num_boost_round=10000,
valid_sets=[dtrain, dvalid],
valid_names=['train', 'valid'],
early_stopping_rounds=100,
verbose_eval=2000,
evals_result=evals_result)
 valid_preds = model.predict(adversarial_x[valid_index])
 train_preds[valid_index] = valid_preds
print('Overall ROC AUC', roc_auc_score(adversarial_y, train_preds))<save_model>
KRR = KernelRidge(alpha=0.6, kernel='polynomial', degree=2, coef0=2.5 )
np.save('weights.npy', predictions.values )<compute_test_metric>
GBoost = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05,
max_depth=4, max_features='sqrt',
min_samples_leaf=15, min_samples_split=10,
loss='huber', random_state =5 )
weights = predictions + 0.1
weights = weights / weights.mean()
def weighted_rmsle(y_true, y_pred, index):
errors =(np.log1p(y_true)- np.log1p(y_pred)) ** 2
errors = errors * weights[index]
return np.sqrt(np.mean(errors))
def rmsle(y_true, y_pred):
errors =(np.log1p(y_true)- np.log1p(y_pred)) ** 2
return np.sqrt(np.mean(errors))<prepare_x_and_y>
model_xgb = xgb.XGBRegressor(colsample_bytree=0.4603, gamma=0.0468,
learning_rate=0.05, max_depth=3,
min_child_weight=1.7817, n_estimators=2200,
reg_alpha=0.4640, reg_lambda=0.8571,
subsample=0.5213, silent=1,
random_state =7, nthread = -1)
x_train = train.iloc[:, 2:].values
x_test = test.iloc[:, 1:].values
y_train = np.log(train['target'])
 _, unique_indices = np.unique(x_train, return_index=True, axis=1)
 variance_greater_zero = x_train.var(axis=0)> 0
mask = np.zeros(x_train.shape[1], dtype=bool)
 mask[unique_indices] = True
mask[variance_greater_zero] = True
x_train = x_train[:, mask]
x_test = x_test[:, mask]
x_train.shape, x_test.shape<normalization>
model_lgb = lgb.LGBMRegressor(objective='regression',num_leaves=5,
learning_rate=0.05, n_estimators=720,
max_bin = 55, bagging_fraction = 0.8,
bagging_freq = 5, feature_fraction = 0.2319,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf =6, min_sum_hessian_in_leaf = 11 )
decomposer = FeatureUnion([ 
('svd', TruncatedSVD(n_components=50, random_state=100)) , 
('ica', FastICA(n_components=20, random_state=100))
])
 decomposed_train = decomposer.fit_transform(x_train)
 decomposed_test = decomposer.transform(x_test )<concatenate>
score = rmsle_cv(lasso)
 print("
Lasso score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
train_matrix = np.hstack([
eng_features.values,
decomposed_train
])
 test_matrix = np.hstack([
eng_features_test.values,
decomposed_test
] )<init_hyperparams>
score = rmsle_cv(ENet)
 print("ElasticNet score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
lgb_params = {
'objective': 'regression',
'metric': 'rmse',
'learning_rate': 0.01,
'num_leaves': 16,
'max_depth': -1,
'min_child_samples': 1,
'max_bin': 300,
'subsample': 1.0,
'subsample_freq': 1,
'colsample_bytree': 0.5,
'min_child_weight': 10,
'reg_lambda': 0.1,
'reg_alpha': 0.0,
'scale_pos_weight': 1,
'zero_as_missing': False,
'num_threads': -1,
}<split>
score = rmsle_cv(KRR)
 print("Kernel Ridge score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
class KFoldByTargetValue(BaseCrossValidator):
def __init__(self, n_splits=3, shuffle=False, random_state=None):
self.n_splits = n_splits
self.shuffle = shuffle
self.random_state = random_state
def _iter_test_indices(self, X, y=None, groups=None):
n_samples = X.shape[0]
indices = np.arange(n_samples)
 sorted_idx_vals = sorted(zip(indices, X), key=lambda x: x[1])
 indices = [idx for idx, val in sorted_idx_vals]
for split_start in range(self.n_splits):
split_indeces = indices[split_start::self.n_splits]
yield split_indeces
def get_n_splits(self, X=None, y=None, groups=None):
return self.n_split<train_model>
score = rmsle_cv(GBoost)
 print("Gradient Boosting score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
cv = KFoldByTargetValue(n_splits=5, shuffle=True, random_state=100)
 train_preds = np.zeros(( len(train)))
 test_preds = np.zeros(( len(test)))
 for i,(train_index, valid_index)in enumerate(cv.split(y_train)) :
print(f'Fold {i}')
 dtrain = lgb.Dataset(train_matrix[train_index],
label=y_train[train_index])
 dvalid = lgb.Dataset(train_matrix[valid_index],
label=y_train[valid_index])
 evals_result = {}
model = lgb.train(lgb_params, dtrain,
num_boost_round=10000,
valid_sets=[dtrain, dvalid],
valid_names=['train', 'valid'],
early_stopping_rounds=100,
verbose_eval=2000,
evals_result=evals_result)
 valid_preds = np.exp(model.predict(train_matrix[valid_index]))
test_preds += np.exp(model.predict(test_matrix)) / cv.n_splits
train_preds[valid_index] = valid_preds
print('RMSLE: ', rmsle(np.exp(y_train[valid_index]), valid_preds))
print('Weighted RMSLE: ', weighted_rmsle(np.exp(y_train[valid_index]), valid_preds, valid_index))
print()
print('Overall RMSLE: ', rmsle(np.exp(y_train), train_preds))
print('Overall Weighted RMSLE: ', weighted_rmsle(np.exp(y_train), train_preds, np.arange(len(train_preds))))<save_to_csv>
score = rmsle_cv(model_xgb)
 print("Xgboost score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
submission = pd.DataFrame()
submission['ID'] = test['ID']
submission['target'] = test_preds
submission.to_csv('submission.csv', index=False )<load_from_csv>
score = rmsle_cv(model_lgb)
 print("LGBM score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
print(os.listdir(".. /input"))
warnings.filterwarnings("ignore")
 train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 test_ID = test['ID']
y_train = train['target']
y_train = np.log1p(y_train)
 train.drop("ID", axis = 1, inplace = True)
 train.drop("target", axis = 1, inplace = True)
 test.drop("ID", axis = 1, inplace = True)
 cols_with_onlyone_val = train.columns[train.nunique() == 1]
train.drop(cols_with_onlyone_val.values, axis=1, inplace=True)
 test.drop(cols_with_onlyone_val.values, axis=1, inplace=True)
 NUM_OF_DECIMALS = 32
train = train.round(NUM_OF_DECIMALS)
 test = test.round(NUM_OF_DECIMALS)
 colsToRemove = []
columns = train.columns
for i in range(len(columns)-1):
v = train[columns[i]].values
dupCols = []
for j in range(i + 1,len(columns)) :
if np.array_equal(v, train[columns[j]].values):
colsToRemove.append(columns[j])
 train.drop(colsToRemove, axis=1, inplace=True)
 test.drop(colsToRemove, axis=1, inplace=True)
 train.shape<train_model>
class AveragingModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, models):
self.models = models
def fit(self, X, y):
self.models_ = [clone(x)for x in self.models]
for model in self.models_:
model.fit(X, y)
 return self
def predict(self, X):
predictions = np.column_stack([
model.predict(X)for model in self.models_
])
 return np.mean(predictions, axis=1 )
NUM_OF_FEATURES = 1000
def rmsle(y, pred):
return np.sqrt(np.mean(np.power(y - pred, 2)))
 x1, x2, y1, y2 = model_selection.train_test_split(
 train, y_train.values, test_size=0.20, random_state=5)
 model = ensemble.RandomForestRegressor(n_jobs=-1, random_state=7)
 model.fit(x1, y1)
 print(rmsle(y2, model.predict(x2)))
 col = pd.DataFrame({'importance': model.feature_importances_, 'feature': train.columns} ).sort_values(
 by=['importance'], ascending=[False])[:NUM_OF_FEATURES]['feature'].values
train = train[col]
test = test[col]
train.shape<categorify>
averaged_models = AveragingModels(models =(ENet, GBoost, KRR, lasso))
score = rmsle_cv(averaged_models)
 print(" Averaged base models score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
THRESHOLD_P_VALUE = 0.01
THRESHOLD_STATISTIC = 0.3
diff_cols = []
for col in train.columns:
statistic, pvalue = ks_2samp(train[col].values, test[col].values)
 if pvalue <= THRESHOLD_P_VALUE and np.abs(statistic)> THRESHOLD_STATISTIC:
diff_cols.append(col)
 for col in diff_cols:
if col in train.columns:
train.drop(col, axis=1, inplace=True)
 test.drop(col, axis=1, inplace=True)
 train.shape<feature_engineering>
class StackingAveragedModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, base_models, meta_model, n_folds=5):
self.base_models = base_models
self.meta_model = meta_model
self.n_folds = n_folds
def fit(self, X, y):
self.base_models_ = [list() for x in self.base_models]
self.meta_model_ = clone(self.meta_model)
 kfold = KFold(n_splits=self.n_folds, shuffle=True, random_state=156)
 out_of_fold_predictions = np.zeros(( X.shape[0], len(self.base_models)))
 for i, model in enumerate(self.base_models):
for train_index, holdout_index in kfold.split(X, y):
instance = clone(model)
 self.base_models_[i].append(instance)
 instance.fit(X[train_index], y[train_index])
 y_pred = instance.predict(X[holdout_index])
 out_of_fold_predictions[holdout_index, i] = y_pred
self.meta_model_.fit(out_of_fold_predictions, y)
 return self
def predict(self, X):
meta_features = np.column_stack([
np.column_stack([model.predict(X)for model in base_models] ).mean(axis=1)
 for base_models in self.base_models_ ])
 return self.meta_model_.predict(meta_features )
ntrain = len(train)
 ntest = len(test)
 tmp = pd.concat([train,test])
 weight =(( train != 0 ).sum() /len(train)).values
tmp_train = train[train!=0]
tmp_test = test[test!=0]
train["weight_count"] =(tmp_train*weight ).sum(axis=1)
 test["weight_count"] =(tmp_test*weight ).sum(axis=1)
 train["count_not0"] =(train != 0 ).sum(axis=1)
 test["count_not0"] =(test != 0 ).sum(axis=1)
 train["sum"] = train.sum(axis=1)
 test["sum"] = test.sum(axis=1)
 train["var"] = tmp_train.var(axis=1)
 test["var"] = tmp_test.var(axis=1)
 train["median"] = tmp_train.median(axis=1)
 test["median"] = tmp_test.median(axis=1)
 train["mean"] = tmp_train.mean(axis=1)
 test["mean"] = tmp_test.mean(axis=1)
 train["std"] = tmp_train.std(axis=1)
 test["std"] = tmp_test.std(axis=1)
 train["max"] = tmp_train.max(axis=1)
 test["max"] = tmp_test.max(axis=1)
 train["min"] = tmp_train.min(axis=1)
 test["min"] = tmp_test.min(axis=1)
 train["skew"] = tmp_train.skew(axis=1)
 test["skew"] = tmp_test.skew(axis=1)
 train["kurtosis"] = tmp_train.kurtosis(axis=1)
 test["kurtosis"] = tmp_test.kurtosis(axis=1)
 del(tmp_train)
 del(tmp_test)
 NUM_OF_COM = 100
transformer = random_projection.SparseRandomProjection(n_components = NUM_OF_COM)
 RP = transformer.fit_transform(tmp)
 rp = pd.DataFrame(RP)
 columns = ["RandomProjection{}".format(i)for i in range(NUM_OF_COM)]
rp.columns = columns
rp_train = rp[:ntrain]
rp_test = rp[ntrain:]
rp_test.index = test.index
train = pd.concat([train,rp_train],axis=1)
 test = pd.concat([test,rp_test],axis=1)
 del(rp_train)
 del(rp_test)
 train.shape<compute_train_metric>
stacked_averaged_models = StackingAveragedModels(base_models =(ENet, GBoost, KRR),
meta_model = lasso)
 score = rmsle_cv(stacked_averaged_models)
 print("Stacking Averaged models score: {:.4f}({:.4f})".format(score.mean() , score.std()))
NUM_FOLDS = 5
def rmsle_cv(model):
kf = KFold(NUM_FOLDS, shuffle=True, random_state=42 ).get_n_splits(train.values)
 rmse= np.sqrt(-cross_val_score(model, train, y_train, scoring="neg_mean_squared_error", cv = kf))
return(rmse)
 class AveragingModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, models):
self.models = models
def fit(self, X, y):
self.models_ = [clone(x)for x in self.models]
for model in self.models_:
model.fit(X, y)
 return self
def predict(self, X):
predictions = np.column_stack([ model.predict(X)for model in self.models_ ])
 return np.mean(predictions, axis=1)
 model_xgb = xgb.XGBRegressor(colsample_bytree=0.055, colsample_bylevel =0.5,
gamma=1.5, learning_rate=0.02, max_depth=32,
objective='reg:linear',booster='gbtree',
min_child_weight=57, n_estimators=1000, reg_alpha=0,
reg_lambda = 0,eval_metric = 'rmse', subsample=0.7,
silent=1, n_jobs = -1, early_stopping_rounds = 14,
random_state =7, nthread = -1)
 model_lgb = lgb.LGBMRegressor(objective='regression',num_leaves=144,
learning_rate=0.005, n_estimators=720, max_depth=13,
metric='rmse',is_training_metric=True,
max_bin = 55, bagging_fraction = 0.8,verbose=-1,
bagging_freq = 5, feature_fraction = 0.9)
 score = rmsle_cv(model_xgb)
 print("Xgboost score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
score = rmsle_cv(model_lgb)
 print("LGBM score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
averaged_models = AveragingModels(models =(model_xgb, model_lgb))
score = rmsle_cv(averaged_models)
 print("averaged score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))<save_to_csv>
def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))
averaged_models.fit(train.values, y_train)
 pred = np.expm1(averaged_models.predict(test.values))
ensemble = pred
sub = pd.DataFrame()
sub['ID'] = test_ID
sub['target'] = ensemble
sub.to_csv('submission.csv',index=False)
<import_modules>
stacked_averaged_models.fit(train.values, y_train)
 stacked_train_pred = stacked_averaged_models.predict(train.values)
 stacked_pred = np.expm1(stacked_averaged_models.predict(test.values))
print(rmsle(y_train, stacked_train_pred))
import pandas as pd
import numpy as np
from sklearn import model_selection
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import RobustScaler
from sklearn.preprocessing import StandardScaler
from sklearn.manifold import TSNE
from sklearn.cluster import KMeans
from sklearn.decomposition import PCA, TruncatedSVD, FastICA
from sklearn.model_selection import KFold
from sklearn.random_projection import GaussianRandomProjection, SparseRandomProjection
from sklearn.preprocessing import normalize
from sklearn.manifold import TSNE
import gc
import seaborn as sns
import lightgbm as lgb
import xgboost as xgb
from catboost import CatBoostRegressor<load_from_csv>
model_xgb.fit(train, y_train)
 xgb_train_pred = model_xgb.predict(train)
 xgb_pred = np.expm1(model_xgb.predict(test))
print(rmsle(y_train, xgb_train_pred))
train_df = pd.read_csv('.. /input/train.csv')
 test_df = pd.read_csv('.. /input/test.csv' )<prepare_x_and_y>
model_lgb.fit(train, y_train)
 lgb_train_pred = model_lgb.predict(train)
 lgb_pred = np.expm1(model_lgb.predict(test.values))
print(rmsle(y_train, lgb_train_pred))
X_train = train_df.drop(["ID", "target"], axis=1)
 y_train = np.log1p(train_df["target"].values)
 X_test = test_df.drop(["ID"], axis=1 )<count_missing_values>

 print('RMSLE score on train data:')
 print(rmsle(y_train,stacked_train_pred*0.70 +
xgb_train_pred*0.15 + lgb_train_pred*0.15))
print("Total Train Features with NaN Values = " + str(train_df.columns[train_df.isnull().sum() != 0].size))
if(train_df.columns[train_df.isnull().sum() != 0].size):
print("Features with NaN => {}".format(list(train_df.columns[train_df.isnull().sum() != 0])))
 train_df[train_df.columns[train_df.isnull().sum() != 0]].isnull().sum().sort_values(ascending = False )<count_values>
ensemble = stacked_pred*0.75 + xgb_pred*0.10 + lgb_pred*0.15
<define_variables><EOS> 
sub = pd.DataFrame()
sub['Id'] = test_ID
sub['SalePrice'] = ensemble
sub.to_csv('submission.csv',index=False )