Kaggle competition: LANL Earthquake Prediction
<SOS> metric: MAE Kaggle data source: lanl-earthquake-prediction<sort_values>
warnings.filterwarnings("ignore" )
def fix_skew(df):
skew_index = []
def show_skew(df):
df_num = df[list(df.select_dtypes(include='number' ).columns)]
skew_features = df_num.apply(lambda x: x.skew() ).sort_values(ascending=False)
 high_skew = skew_features[skew_features > 0.6]
nonlocal skew_index
skew_index = high_skew.index
print("{} features have skew > 0.6 :".format(high_skew.shape[0]))
skewness = pd.DataFrame({'Skew' :high_skew})
 print(skew_features.head(10))
show_skew(df)
 for i in skew_index:
df[i] = boxcox1p(df[i], boxcox_normmax(df[i] + 1))
show_skew(df )<feature_engineering>
%%time
train_df = pd.read_csv(os.path.join(PATH,'train.csv'),
dtype={'acoustic_data': np.int16, 'time_to_failure': np.float32} )
combined['Has2ndFlr'] = combined['2ndFlrSF'].apply(lambda x : 1 if x > 0 else 0)
 combined['HasWoodDeck'] = combined['WoodDeckSF'].apply(lambda x : 1 if x > 0 else 0)
 combined['HasOpenPorch'] = combined['OpenPorchSF'].apply(lambda x : 1 if x > 0 else 0)
 combined['HasEnclosedPorch'] = combined['EnclosedPorch'].apply(lambda x : 1 if x > 0 else 0)
 combined['Has3SsnPorch'] = combined['3SsnPorch'].apply(lambda x : 1 if x > 0 else 0)
 combined['HasScreenPorch'] = combined['ScreenPorch'].apply(lambda x : 1 if x > 0 else 0 )<count_values>
train_df_save = train_df.copy
def binarize(column, data=combined):
print(combined[column].value_counts())
 combined[column] = combined[column].apply(lambda x : 0 if x == 'None' else 1)
 print(combined[column].value_counts() )<feature_engineering>
train_df.acoustic_data = np.log1p(abs(train_df.acoustic_data))
combined['HouseQualAdd'] = combined['OverallQual'] + combined['OverallCond']
combined['HouseQualProd'] = combined['OverallQual'] * combined['OverallCond']<feature_engineering>
gc.collect()
combined['TotalSqrFt'] =(combined['BsmtFinSF1'] + combined['BsmtFinSF2']
+ combined['1stFlrSF'] + combined['2ndFlrSF'])
 combined['TotalSF'] =(combined['TotalBsmtSF'] + combined['1stFlrSF'] + combined['2ndFlrSF'])
 combined['TotalPorchSF'] =(combined['WoodDeckSF'] + combined['OpenPorchSF']
+ combined['EnclosedPorch'] + combined['3SsnPorch']
+ combined['ScreenPorch'])
 combined['TotalBath'] =(combined['FullBath']
+(0.5 * combined['HalfBath'])
 + combined['BsmtFullBath']
+(0.5 * combined['BsmtHalfBath']))<drop_column>
test = pd.read_csv('.. /input/test/seg_00030f.csv')
 test.shape
combined.drop(['Street', 'Utilities'], axis=1, inplace=True )<categorify>
rows = 150000
segments = int(np.floor(train_df.shape[0] * 8 / rows))
print("Number of segments: ", segments )
combined_encoded = pd.get_dummies(combined ).reset_index(drop=True )<count_duplicates>
def add_trend_feature(arr, abs_values=False):
idx = np.array(range(len(arr)))
 if abs_values:
arr = np.abs(arr)
 lr = LinearRegression()
lr.fit(idx.reshape(-1, 1), arr)
 return lr.coef_[0]
def classic_sta_lta(x, length_sta, length_lta):
sta = np.cumsum(x ** 2)
 sta = np.require(sta, dtype=np.float)
 lta = sta.copy()
sta[length_sta:] = sta[length_sta:] - sta[:-length_sta]
sta /= length_sta
lta[length_lta:] = lta[length_lta:] - lta[:-length_lta]
lta /= length_lta
sta[:length_lta - 1] = 0
dtiny = np.finfo(0.0 ).tiny
idx = lta < dtiny
lta[idx] = dtiny
return sta / lta
combined_encoded = combined_encoded.loc[:,~combined_encoded.columns.duplicated() ]<prepare_x_and_y>
train_X = pd.DataFrame(index=range(segments), dtype=np.float32)
 train_y = pd.DataFrame(index=range(segments), dtype=np.float32, columns=['time_to_failure'] )
y_test, X_test =(combined_encoded[len(train):]["SalePrice"],
combined_encoded.drop(["SalePrice"], axis=1)[len(train):] )<prepare_x_and_y>
def create_features(seg_id, seg, X):
xc = pd.Series(seg['acoustic_data'].values)
 zc = np.fft.fft(xc)
 X.loc[seg_id, 'mean'] = xc.mean()
X.loc[seg_id, 'std'] = xc.std()
X.loc[seg_id, 'max'] = xc.max()
realFFT = np.real(zc)
 imagFFT = np.imag(zc)
 X.loc[seg_id, 'RIsquare'] = np.mean(np.square(realFFT)+ np.square(imagFFT))
X.loc[seg_id, 'RImsquare'] = np.square(realFFT.mean())+ np.square(imagFFT.mean())
 X.loc[seg_id, 'Rmean'] = realFFT.mean()
X.loc[seg_id, 'Rstd'] = realFFT.std()
X.loc[seg_id, 'Rmax'] = realFFT.max()
X.loc[seg_id, 'Rmin'] = realFFT.min()
X.loc[seg_id, 'Imean'] = imagFFT.mean()
X.loc[seg_id, 'Istd'] = imagFFT.std()
X.loc[seg_id, 'Imax'] = imagFFT.max()
X.loc[seg_id, 'Imin'] = imagFFT.min()
X.loc[seg_id, 'Rmean_last_5000'] = realFFT[-5000:].mean()
X.loc[seg_id, 'Rstd__last_5000'] = realFFT[-5000:].std()
X.loc[seg_id, 'Rmax_last_5000'] = realFFT[-5000:].max()
X.loc[seg_id, 'Rmin_last_5000'] = realFFT[-5000:].min()
X.loc[seg_id, 'Rmean_last_15000'] = realFFT[-15000:].mean()
X.loc[seg_id, 'Rstd_last_15000'] = realFFT[-15000:].std()
X.loc[seg_id, 'Rmax_last_15000'] = realFFT[-15000:].max()
X.loc[seg_id, 'Rmin_last_15000'] = realFFT[-15000:].min()
X.loc[seg_id, 'mean_change_abs'] = np.mean(np.diff(xc))
X.loc[seg_id, 'mean_change_rate'] = np.mean(np.nonzero(( np.diff(xc)/ xc[:-1])) [0])
 X.loc[seg_id, 'abs_max'] = np.abs(xc ).max()
X.loc[seg_id, 'abs_min'] = np.abs(xc ).min()
X.loc[seg_id, 'std_first_50000'] = xc[:50000].std()
X.loc[seg_id, 'std_last_50000'] = xc[-50000:].std()
X.loc[seg_id, 'std_first_10000'] = xc[:10000].std()
X.loc[seg_id, 'std_last_10000'] = xc[-10000:].std()
X.loc[seg_id, 'avg_first_50000'] = xc[:50000].mean()
X.loc[seg_id, 'avg_last_50000'] = xc[-50000:].mean()
X.loc[seg_id, 'avg_first_10000'] = xc[:10000].mean()
X.loc[seg_id, 'avg_last_10000'] = xc[-10000:].mean()
X.loc[seg_id, 'min_first_50000'] = xc[:50000].min()
X.loc[seg_id, 'min_last_50000'] = xc[-50000:].min()
X.loc[seg_id, 'min_first_10000'] = xc[:10000].min()
X.loc[seg_id, 'min_last_10000'] = xc[-10000:].min()
X.loc[seg_id, 'max_first_50000'] = xc[:50000].max()
X.loc[seg_id, 'max_last_50000'] = xc[-50000:].max()
X.loc[seg_id, 'max_first_10000'] = xc[:10000].max()
X.loc[seg_id, 'max_last_10000'] = xc[-10000:].max()
X.loc[seg_id, 'max_to_min_diff'] = xc.max() - np.abs(xc.min())
 X.loc[seg_id, 'count_big'] = len(xc[np.abs(xc)> 500])
 X.loc[seg_id, 'sum'] = xc.sum()
X.loc[seg_id, 'mean_change_rate_first_50000'] = np.mean(np.nonzero(( np.diff(xc[:50000])/ xc[:50000][:-1])) [0])
 X.loc[seg_id, 'mean_change_rate_last_50000'] = np.mean(np.nonzero(( np.diff(xc[-50000:])/ xc[-50000:][:-1])) [0])
 X.loc[seg_id, 'mean_change_rate_first_10000'] = np.mean(np.nonzero(( np.diff(xc[:10000])/ xc[:10000][:-1])) [0])
 X.loc[seg_id, 'mean_change_rate_last_10000'] = np.mean(np.nonzero(( np.diff(xc[-10000:])/ xc[-10000:][:-1])) [0])
 X.loc[seg_id, 'q95'] = np.quantile(xc, 0.95)
 X.loc[seg_id, 'q99'] = np.quantile(xc, 0.99)
 X.loc[seg_id, 'q05'] = np.quantile(xc, 0.05)
 X.loc[seg_id, 'q01'] = np.quantile(xc, 0.01)
 X.loc[seg_id, 'q999'] = np.quantile(xc,0.999)
 X.loc[seg_id, 'q001'] = np.quantile(xc,0.001)
 X.loc[seg_id, 'abs_q95'] = np.quantile(np.abs(xc), 0.95)
 X.loc[seg_id, 'abs_q99'] = np.quantile(np.abs(xc), 0.99)
 X.loc[seg_id, 'abs_q05'] = np.quantile(np.abs(xc), 0.05)
 X.loc[seg_id, 'abs_q01'] = np.quantile(np.abs(xc), 0.01)
 X.loc[seg_id, 'trend'] = add_trend_feature(xc)
 X.loc[seg_id, 'abs_trend'] = add_trend_feature(xc, abs_values=True)
 X.loc[seg_id, 'abs_mean'] = np.abs(xc ).mean()
X.loc[seg_id, 'abs_std'] = np.abs(xc ).std()
X.loc[seg_id, 'mad'] = xc.mad()
X.loc[seg_id, 'Moving_average_700_mean'] = xc.rolling(window=700 ).mean().mean(skipna=True)
 X.loc[seg_id, 'Moving_average_1500_mean'] = xc.rolling(window=1500 ).mean().mean(skipna=True)
 X.loc[seg_id, 'Moving_average_3000_mean'] = xc.rolling(window=3000 ).mean().mean(skipna=True)
 X.loc[seg_id, 'Moving_average_6000_mean'] = xc.rolling(window=6000 ).mean().mean(skipna=True)
 ewma = pd.Series.ewm
X.loc[seg_id, 'exp_Moving_average_300_mean'] =(ewma(xc, span=300 ).mean() ).mean(skipna=True)
 X.loc[seg_id, 'exp_Moving_average_3000_mean'] = ewma(xc, span=3000 ).mean().mean(skipna=True)
 X.loc[seg_id, 'exp_Moving_average_30000_mean'] = ewma(xc, span=6000 ).mean().mean(skipna=True)
 no_of_std = 2
X.loc[seg_id, 'MA_700MA_std_mean'] = xc.rolling(window=700 ).std().mean()
X.loc[seg_id,'MA_700MA_BB_high_mean'] =(X.loc[seg_id, 'Moving_average_700_mean'] + no_of_std * X.loc[seg_id, 'MA_700MA_std_mean'] ).mean()
X.loc[seg_id,'MA_700MA_BB_low_mean'] =(X.loc[seg_id, 'Moving_average_700_mean'] - no_of_std * X.loc[seg_id, 'MA_700MA_std_mean'] ).mean()
X.loc[seg_id, 'MA_400MA_std_mean'] = xc.rolling(window=400 ).std().mean()
X.loc[seg_id,'MA_400MA_BB_high_mean'] =(X.loc[seg_id, 'Moving_average_700_mean'] + no_of_std * X.loc[seg_id, 'MA_400MA_std_mean'] ).mean()
X.loc[seg_id,'MA_400MA_BB_low_mean'] =(X.loc[seg_id, 'Moving_average_700_mean'] - no_of_std * X.loc[seg_id, 'MA_400MA_std_mean'] ).mean()
X.loc[seg_id, 'MA_1000MA_std_mean'] = xc.rolling(window=1000 ).std().mean()
X.loc[seg_id, 'iqr'] = np.subtract(*np.percentile(xc, [75, 25]))
X.loc[seg_id, 'ave10'] = stats.trim_mean(xc, 0.1)
 for windows in [5, 10, 50, 100, 500, 1000, 5000, 10000]:
x_roll_std = xc.rolling(windows ).std().dropna().values
x_roll_mean = xc.rolling(windows ).mean().dropna().values
X.loc[seg_id, 'ave_roll_std_' + str(windows)] = x_roll_std.mean()
X.loc[seg_id, 'std_roll_std_' + str(windows)] = x_roll_std.std()
X.loc[seg_id, 'max_roll_std_' + str(windows)] = x_roll_std.max()
X.loc[seg_id, 'min_roll_std_' + str(windows)] = x_roll_std.min()
X.loc[seg_id, 'q01_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.01)
 X.loc[seg_id, 'q05_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.05)
 X.loc[seg_id, 'q95_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.95)
 X.loc[seg_id, 'q99_roll_std_' + str(windows)] = np.quantile(x_roll_std, 0.99)
 X.loc[seg_id, 'av_change_abs_roll_std_' + str(windows)] = np.mean(np.diff(x_roll_std))
X.loc[seg_id, 'av_change_rate_roll_std_' + str(windows)] = np.mean(np.nonzero(( np.diff(x_roll_std)/ x_roll_std[:-1])) [0])
 X.loc[seg_id, 'abs_max_roll_std_' + str(windows)] = np.abs(x_roll_std ).max()
X.loc[seg_id, 'ave_roll_mean_' + str(windows)] = x_roll_mean.mean()
X.loc[seg_id, 'std_roll_mean_' + str(windows)] = x_roll_mean.std()
X.loc[seg_id, 'max_roll_mean_' + str(windows)] = x_roll_mean.max()
X.loc[seg_id, 'min_roll_mean_' + str(windows)] = x_roll_mean.min()
X.loc[seg_id, 'q01_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.01)
 X.loc[seg_id, 'q05_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.05)
 X.loc[seg_id, 'q95_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.95)
 X.loc[seg_id, 'q99_roll_mean_' + str(windows)] = np.quantile(x_roll_mean, 0.99)
 X.loc[seg_id, 'av_change_abs_roll_mean_' + str(windows)] = np.mean(np.diff(x_roll_mean))
X.loc[seg_id, 'av_change_rate_roll_mean_' + str(windows)] = np.mean(np.nonzero(( np.diff(x_roll_mean)/ x_roll_mean[:-1])) [0])
 X.loc[seg_id, 'abs_max_roll_mean_' + str(windows)] = np.abs(x_roll_mean ).max()
y_train_full, X_train_full =(combined_encoded[:len(train)]["SalePrice"],
combined_encoded.drop(["SalePrice"], axis=1)[:len(train)] )<split>
for seg_id in tqdm_notebook(range(segments)) :
seg = train_df.iloc[seg_id*int(rows/8):seg_id*int(rows/8)+rows]
create_features(seg_id, seg, train_X)
 train_y.loc[seg_id, 'time_to_failure'] = seg['time_to_failure'].values[-1]
X_train, X_valid, y_train, y_valid = train_test_split(
 X_train_full, y_train_full, test_size=0.20, random_state=42 )<import_modules>
train_X_save = train_X.copy
train_y_save = train_y.copy
train_y.head()
train_y = train_y**1.0
train_X.head()
from sklearn.ensemble import RandomForestRegressor
from xgboost import XGBRegressor
from lightgbm import LGBMRegressor
from sklearn.ensemble import GradientBoostingRegressor
from sklearn.svm import SVR
from sklearn.linear_model import RidgeCV, LassoCV, ElasticNetCV
from mlxtend.regressor import StackingCVRegressor
from sklearn.model_selection import KFold, cross_val_score
from sklearn.pipeline import make_pipeline
from sklearn.metrics import mean_squared_error
import hyperopt
from hyperopt import hp
from hyperopt import fmin, tpe, hp, anneal, Trials
from sklearn.preprocessing import RobustScaler<compute_test_metric>
train_X.to_csv('train_X.csv',index=False)
 train_y.to_csv('train_y.csv',index=False )
def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))<compute_train_metric>
scaler = StandardScaler()
scaler.fit(train_X)
 scaled_train_X = pd.DataFrame(scaler.transform(train_X), columns=train_X.columns)
kfolds = KFold(n_splits=10, random_state=42, shuffle=True)
 def cv_rmse(model, X=X_train_full):
rmse = np.sqrt(-cross_val_score(model, X, y_train_full,
scoring='neg_mean_squared_error', cv=kfolds))
return(rmse )<compute_train_metric>
temp = train_X.copy
def mse_cv(params, cv=kfolds, X=X_train_full, y=y_train_full):
params = {'max_depth': int(params['max_depth']),
'learning_rate': params['learning_rate'],
'gamma': params['gamma'],
'colsample_bytree': params['colsample_bytree'],
'subsample': params['subsample']
}
model = XGBRegressor(objective='reg:linear', n_estimators=200,
random_state=42,
**params)
 score = np.sqrt(-cross_val_score(model, X, y, cv=cv, scoring="neg_mean_squared_error", n_jobs=-1 ).mean())
 return score<init_hyperparams>
submission = pd.read_csv('.. /input/sample_submission.csv', index_col='seg_id')
 test_X = pd.DataFrame(columns=train_X.columns, dtype=np.float64, index=submission.index )
space={'max_depth' : hp.quniform('max_depth', 2, 10, 1),
'learning_rate': hp.loguniform('learning_rate', -5, 0),
'gamma': hp.loguniform('gamma', -1, 0),
'colsample_bytree': hp.loguniform('colsample_bytree', -1, 0),
'subsample': hp.loguniform('subsample', -1, 0)
 }
trials = Trials()
best=fmin(fn=mse_cv,
space=space,
algo=tpe.suggest,
max_evals=200,
trials=trials,
rstate=np.random.RandomState(42)
 )<compute_test_metric>
for seg_id in tqdm_notebook(test_X.index):
seg = pd.read_csv('.. /input/test/' + seg_id + '.csv')
 seg.acoustic_data =np.log1p(abs(seg.acoustic_data))
create_features(seg_id, seg, test_X )
xgb_reg = XGBRegressor(random_state=42, n_estimators=5000,
max_depth=int(best['max_depth']),learning_rate=best['learning_rate'],
gamma=best['gamma'], colsample_bytree=best['colsample_bytree'],
subsample=best['subsample'], objective='reg:linear',
nthread=-1, scale_pos_weight=1)
 print(np.mean(cv_rmse(xgb_reg)))
 print("Best MSE {:.3f} params {}".format(mse_cv(best), best))<choose_model_class>
test_X.to_csv('test_X.csv',index=False )
xgb_reg = XGBRegressor(random_state=42,
n_estimators=7000,
max_depth=2,
learning_rate=0.138170657,
gamma=0.38498075,
colsample_bytree=0.599437614,
objective='reg:linear',
nthread=-1,
scale_pos_weight=1,
subsample=0.534382267 )<choose_model_class>
scaled_test_X = pd.DataFrame(scaler.transform(test_X), columns=test_X.columns)
 scaled_test_X.values[1117]
lgbm_reg = LGBMRegressor(random_state=42,
n_estimators=7000,
num_leaves=3,
learning_rate=0.099384860,
bagging_seed=5,
feature_fraction_seed=5,
bagging_fraction=0.407679661,
feature_fraction=0.563479974,
min_sum_hessian_in_leaf=20 )<define_search_model>
n_fold = 5
def mae_cv(model):
folds = KFold(n_splits=n_fold, shuffle=True, random_state=42 ).get_n_splits(scaled_train_X.values)
 mae = -cross_val_score(model, scaled_train_X.values, train_y, scoring="neg_mean_absolute_error",
verbose=0,
cv=folds)
 return mae
svr_reg = make_pipeline(RobustScaler() , SVR(C=11,
gamma=0.006742321288,
epsilon=0.010861719298))<choose_model_class>
gb_reg = GradientBoostingRegressor(max_depth=21,
min_samples_leaf=11,
min_samples_split=8,
learning_rate=0.038954801112873964,
loss='huber', max_features='sqrt',
n_estimators=7000, random_state=42 )<define_search_space>
ridge_alphas = [1e-10, 1e-8, 1e-5, 9e-4, 7e-4, 5e-4, 3e-4, 1e-4, 1e-3, 5e-2, 1e-2,
0.1, 0.3, 1, 3, 5, 10, 15, 18, 20, 30, 50, 75, 100]
lasso_alphas = [5e-5, 0.0001, 0.0002, 0.0003, 0.0004, 0.0005, 0.0006, 0.0007, 0.0008]
elastic_alphas = [0.0001, 0.0002, 0.0003, 0.0004, 0.0005, 0.0006, 0.0007, 0.0008, 0.0009, 0.001]
elastic_l1ratio = [0.5, 0.6, 0.7, 0.75, 0.8, 0.85, 0.9, 0.95, 0.97, 0.99, 1]<define_search_model>
%%time
rf_model = RandomForestRegressor(n_estimators=120, n_jobs=-1, min_samples_leaf=1,
max_features = "auto",max_depth=15,)
 ".format(score.mean() , score.std()))
rf_model
ridge_reg = make_pipeline(RobustScaler() , RidgeCV(alphas=ridge_alphas, cv=kfolds))
lasso_reg = make_pipeline(RobustScaler() , LassoCV(max_iter=1e7, alphas=lasso_alphas,
random_state=42, cv=kfolds))
elasticnet_reg = make_pipeline(RobustScaler() , ElasticNetCV(max_iter=1e7,
alphas=elastic_alphas,
cv=kfolds,
l1_ratio=elastic_l1ratio))<choose_model_class>
%%time
params = {'loss_function':'MAE',}
cat_model = CatBoostRegressor(iterations=1000, eval_metric='MAE', verbose=False, **params)
 ".format(score.mean() , score.std()))
cat_model
stack_gen = StackingCVRegressor(regressors=(ridge_reg, lasso_reg, elasticnet_reg, xgb_reg, lgbm_reg, gb_reg, svr_reg),
meta_regressor=xgb_reg,
use_features_in_secondary=True )<compute_test_metric>
class AveragingModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, models):
self.models = models
def fit(self, X, y):
self.models_ = [clone(x)for x in self.models]
for model in self.models_:
model.fit(X, y)
 return self
def predict(self, X):
predictions = np.column_stack([
model.predict(X)for model in self.models_
])
 return np.mean(predictions, axis=1 )
scores = {}
score = cv_rmse(lgbm_reg)
 print("lgbm_reg: {:.4f}({:.4f})".format(score.mean() , score.std()))
scores['lgbm'] =(score.mean() , score.std())
 score = cv_rmse(xgb_reg)
 print("xgb_reg: {:.4f}({:.4f})".format(score.mean() , score.std()))
scores['xgb'] =(score.mean() , score.std())
 score = cv_rmse(svr_reg)
 print("SVR: {:.4f}({:.4f})".format(score.mean() , score.std()))
scores['svr'] =(score.mean() , score.std())
 score = cv_rmse(ridge_reg)
 print("ridge_reg: {:.4f}({:.4f})".format(score.mean() , score.std()))
scores['ridge'] =(score.mean() , score.std())
 score = cv_rmse(lasso_reg)
 print("lasso_reg: {:.4f}({:.4f})".format(score.mean() , score.std()))
scores['lasso'] =(score.mean() , score.std())
 score = cv_rmse(elasticnet_reg)
 print("elasticnet_reg: {:.4f}({:.4f})".format(score.mean() , score.std()))
scores['elasticnet'] =(score.mean() , score.std())
 score = cv_rmse(gb_reg)
 print("gb_reg: {:.4f}({:.4f})".format(score.mean() , score.std()))
scores['gb'] =(score.mean() , score.std() )<train_model>
class StackingAveragedModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, base_models, meta_model, n_folds=5):
self.base_models = base_models
self.meta_model = meta_model
self.n_folds = n_folds
def fit(self, X, y):
print(type(X))
self.base_models_ = [list() for x in self.base_models]
self.meta_model_ = clone(self.meta_model)
 kfold = KFold(n_splits=self.n_folds, shuffle=True, random_state=156)
 print(KFold)
 out_of_fold_predictions = np.zeros(( X.shape[0], len(self.base_models)))
 for i, model in enumerate(self.base_models):
for train_index, holdout_index in kfold.split(X, y):
instance = clone(model)
 self.base_models_[i].append(instance)
 instance.fit(X.iloc[train_index], y.iloc[train_index])
 y_pred = instance.predict(X.iloc[holdout_index])
 out_of_fold_predictions[holdout_index, i] = y_pred
self.meta_model_.fit(out_of_fold_predictions, y)
 return self
def predict(self, X):
meta_features = np.column_stack([
np.column_stack([model.predict(X)for model in base_models] ).mean(axis=1)
 for base_models in self.base_models_ ])
 return self.meta_model_.predict(meta_features )
%%time
print('START Fit')
 print('stack_gen')
 stack_gen_model = stack_gen.fit(np.array(X_train_full), np.array(y_train_full))
print('elasticnet')
 elastic_reg_model = elasticnet_reg.fit(X_train_full, y_train_full)
 print('Lasso')
 lasso_reg_model = lasso_reg.fit(X_train_full, y_train_full)
 print('Ridge')
 ridge_reg_model = ridge_reg.fit(X_train_full, y_train_full)
 print('Svr')
 svr_reg_model = svr_reg.fit(X_train_full, y_train_full)
 print('GradientBoosting')
 gb_reg_model = gb_reg.fit(X_train_full, y_train_full)
 print('xgboost')
 xgb_reg_model = xgb_reg.fit(X_train_full, y_train_full)
 print('lightgbm')
 lgbm_reg_model = lgbm_reg.fit(X_train_full, y_train_full)
 print('END FIT' )<predict_on_test>
class StackingCVRegressorRetrained(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, regressors, meta_regressor, n_folds=5, use_features_in_secondary=False):
self.regressors = regressors
self.meta_regressor = meta_regressor
self.n_folds = n_folds
self.use_features_in_secondary = use_features_in_secondary
def fit(self, X, y):
self.regr_ = [clone(x)for x in self.regressors]
self.meta_regr_ = clone(self.meta_regressor)
 kfold = KFold(n_splits=self.n_folds, shuffle=True)
 out_of_fold_predictions = np.zeros(( X.shape[0], len(self.regressors)))
 for i, regr in enumerate(self.regr_):
for train_idx, holdout_idx in kfold.split(X, y):
instance = clone(regr)
 instance.fit(X[train_idx], y[train_idx])
 out_of_fold_predictions[holdout_idx, i] = instance.predict(X[holdout_idx])
 if self.use_features_in_secondary:
self.meta_regr_.fit(np.hstack(( X, out_of_fold_predictions)) , y)
 else:
self.meta_regr_.fit(out_of_fold_predictions, y)
 for regr in self.regr_:
regr.fit(X, y)
 return self
def predict(self, X):
meta_features = np.column_stack([
regr.predict(X)for regr in self.regr_
])
 if self.use_features_in_secondary:
return self.meta_regr_.predict(np.hstack(( X, meta_features)))
 else:
return self.meta_regr_.predict(meta_features )
def blender(X):
return(( 0.1 * elastic_reg_model.predict(X)) + \ 
(0.1 * lasso_reg_model.predict(X)) + \ 
(0.1 * ridge_reg_model.predict(X)) + \ 
(0.1 * svr_reg_model.predict(X)) + \ 
(0.1 * gb_reg_model.predict(X)) + \ 
(0.1 * xgb_reg_model.predict(X)) + \ 
(0.1 * lgbm_reg_model.predict(X)) + \ 
(0.3 * stack_gen_model.predict(np.array(X))))<compute_test_metric>
%%time
averaged_models = AveragingModels(models =(rf_model,cat_model))
score = mae_cv(averaged_models)
 print(" Averaged base models score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
print('RMSLE score on train data:')
 print(rmsle(y_train_full, blender(X_train_full)) )<load_from_csv>
averaged_models.fit(scaled_train_X.values, train_y)
 averaged_train_predict = averaged_models.predict(scaled_train_X.values)
 print(mean_absolute_error(train_y, averaged_train_predict))
print('Predict submission')
 submission = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/sample_submission.csv")
 submission.iloc[:,1] = np.floor(np.expm1(blender(X_test)) )<save_to_csv>
averaged_prediction = np.zeros(len(scaled_test_X))
averaged_prediction += averaged_models.predict(scaled_test_X.values)
 averaged_prediction
submission.to_csv("submission.csv", index=False )<save_to_csv>
<set_options><EOS> 
submission.time_to_failure = averaged_prediction
submission.to_csv('submission_averaged_cat_rf_8.csv',index=True)