Kaggle competition: House Prices - Advanced Regression Techniques
def preprocess(reduce_train, reduce_test):

 for df in [reduce_train, reduce_test]:
df['installation_session_count'] = df.groupby(['installation_id'])['Clip'].transform('count')
 df['installation_duration_mean'] = df.groupby(['installation_id'])['duration_mean'].transform('mean')
 df['installation_title_nunique'] = df.groupby(['installation_id'])['session_title'].transform('nunique')
 df['sum_event_code_count'] = df[[2050, 4100, 4230, 5000, 4235, 2060, 4110, 5010, 2070, 2075, 2080, 2081, 2083, 3110, 4010, 3120, 3121, 4020, 4021,
4022, 4025, 4030, 4031, 3010, 4035, 4040, 3020, 3021, 4045, 2000, 4050, 2010, 2020, 4070, 2025, 2030, 4080, 2035,
2040, 4090, 4220, 4095]].sum(axis = 1)
 df['installation_event_code_count_mean'] = df.groupby(['installation_id'])['sum_event_code_count'].transform('mean')
 return reduce_train, reduce_test<categorify>
train = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv')
 test = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv' )
train, test, train_labels, specs, sample_submission = read_data()
train, test, train_labels, win_code, list_of_user_activities, list_of_event_code, activities_labels, assess_titles, list_of_event_id, all_title_event_code = encode_title(train, test, train_labels)
 reduce_train = None
reduce_test = None
reduce_train, reduce_test, categoricals = get_train_and_test(train, test)
 reduce_train, reduce_test = preprocess(reduce_train, reduce_test )<categorify>
train_ID = train['Id']
test_ID = test['Id']
train.drop("Id", axis = 1, inplace = True)
 test.drop("Id", axis = 1, inplace = True )
@jit
def qwk(a1, a2):

 max_rat = 3
a1 = np.asarray(a1, dtype=int)
 a2 = np.asarray(a2, dtype=int)
 hist1 = np.zeros(( max_rat + 1,))
hist2 = np.zeros(( max_rat + 1,))
o = 0
for k in range(a1.shape[0]):
i, j = a1[k], a2[k]
hist1[i] += 1
hist2[j] += 1
o +=(i - j)*(i - j)
 e = 0
for i in range(max_rat + 1):
for j in range(max_rat + 1):
e += hist1[i] * hist2[j] *(i - j)*(i - j)
 e = e / a1.shape[0]
return 1 - o / e
def eval_qwk_lgb(y_true, y_pred):

 y_pred = y_pred.reshape(len(np.unique(y_true)) , -1 ).argmax(axis=0)
 return 'cappa', qwk(y_true, y_pred), True
def eval_qwk_lgb_regr(y_true, y_pred):

 y_pred[y_pred <= 1.12232214] = 0
y_pred[np.where(np.logical_and(y_pred > 1.12232214, y_pred <= 1.73925866)) ] = 1
y_pred[np.where(np.logical_and(y_pred > 1.73925866, y_pred <= 2.22506454)) ] = 2
y_pred[y_pred > 2.22506454] = 3
return 'cappa', qwk(y_true, y_pred), True
def eval_qwk_xgb(y_pred, y_true):

 y_true = y_true.get_label()
y_pred = y_pred.argmax(axis=1)
 return 'cappa', -qwk(y_true, y_pred)
 class MainTransformer(BaseEstimator, TransformerMixin):
def __init__(self, convert_cyclical: bool = False, create_interactions: bool = False, n_interactions: int = 20):

 self.convert_cyclical = convert_cyclical
self.create_interactions = create_interactions
self.feats_for_interaction = None
self.n_interactions = n_interactions
def fit(self, X, y=None):
if self.create_interactions:
self.feats_for_interaction = [col for col in X.columns if 'sum' in col
or 'mean' in col or 'max' in col or 'std' in col
or 'attempt' in col]
self.feats_for_interaction1 = np.random.choice(self.feats_for_interaction, self.n_interactions)
 self.feats_for_interaction2 = np.random.choice(self.feats_for_interaction, self.n_interactions)
 return self
def transform(self, X, y=None):
data = copy.deepcopy(X)
 if self.create_interactions:
for col1 in self.feats_for_interaction1:
for col2 in self.feats_for_interaction2:
data[f'{col1}_int_{col2}'] = data[col1] * data[col2]
if self.convert_cyclical:
data['timestampHour'] = np.sin(2 * np.pi * data['timestampHour'] / 23.0)
 data['timestampMonth'] = np.sin(2 * np.pi * data['timestampMonth'] / 23.0)
 data['timestampWeek'] = np.sin(2 * np.pi * data['timestampWeek'] / 23.0)
 data['timestampMinute'] = np.sin(2 * np.pi * data['timestampMinute'] / 23.0)
 return data
def fit_transform(self, X, y=None, **fit_params):
data = copy.deepcopy(X)
 self.fit(data)
 return self.transform(data)
 class FeatureTransformer(BaseEstimator, TransformerMixin):
def __init__(self, main_cat_features: list = None, num_cols: list = None):

 self.main_cat_features = main_cat_features
self.num_cols = num_cols
def fit(self, X, y=None):
return self
def transform(self, X, y=None):
data = copy.deepcopy(X)
 return data
def fit_transform(self, X, y=None, **fit_params):
data = copy.deepcopy(X)
 self.fit(data)
 return self.transform(data)
<train_model>
ntrain = train.shape[0]
ntest = test.shape[0]
y_train = train.SalePrice.values
data = pd.concat(( train, test)).reset_index(drop=True )
class LGBWrapper_regr(object):

 def __init__(self):
self.model = lgb.LGBMRegressor()
def fit(self, X_train, y_train, X_valid=None, y_valid=None, X_holdout=None, y_holdout=None, params=None):
if params['objective'] == 'regression':
eval_metric = eval_qwk_lgb_regr
else:
eval_metric = 'auc'
eval_set = [(X_train, y_train)]
eval_names = ['train']
self.model = self.model.set_params(**params)
 if X_valid is not None:
eval_set.append(( X_valid, y_valid))
eval_names.append('valid')
 if X_holdout is not None:
eval_set.append(( X_holdout, y_holdout))
eval_names.append('holdout')
 if 'cat_cols' in params.keys() :
cat_cols = [col for col in params['cat_cols'] if col in X_train.columns]
if len(cat_cols)> 0:
categorical_columns = params['cat_cols']
else:
categorical_columns = 'auto'
else:
categorical_columns = 'auto'
print("fit : ", " eval_names=", eval_names, " eval_metric=", eval_metric, " categorical_feature=", categorical_columns)
 self.model.fit(X=X_train, y=y_train,
eval_set=eval_set, eval_names=eval_names, eval_metric=eval_metric,
verbose=params['verbose'], early_stopping_rounds=params['early_stopping_rounds'],
categorical_feature=categorical_columns)
 self.best_score_ = self.model.best_score_
self.feature_importances_ = self.model.feature_importances_
def predict(self, X_test):
return self.model.predict(X_test, num_iteration=self.model.best_iteration_)
<train_model>
data_na =(data.isnull().sum() / len(data)) * 100
data_na = data_na.drop(data_na[data_na == 0].index ).sort_values(ascending=False)[:30]
missing_data = pd.DataFrame({'Missing Ratio' :data_na})
 missing_data.head(20 )
class LGBWrapper(object):

 def __init__(self):
self.model = lgb.LGBMClassifier()
def fit(self, X_train, y_train, X_valid=None, y_valid=None, X_holdout=None, y_holdout=None, params=None):
eval_set = [(X_train, y_train)]
eval_names = ['train']
self.model = self.model.set_params(**params)
 if X_valid is not None:
eval_set.append(( X_valid, y_valid))
eval_names.append('valid')
 if X_holdout is not None:
eval_set.append(( X_holdout, y_holdout))
eval_names.append('holdout')
 if 'cat_cols' in params.keys() :
cat_cols = [col for col in params['cat_cols'] if col in X_train.columns]
if len(cat_cols)> 0:
categorical_columns = params['cat_cols']
else:
categorical_columns = 'auto'
else:
categorical_columns = 'auto'
self.model.fit(X=X_train, y=y_train,
eval_set=eval_set, eval_names=eval_names, eval_metric=eval_qwk_lgb,
verbose=params['verbose'], early_stopping_rounds=params['early_stopping_rounds'],
categorical_feature=categorical_columns)
 self.best_score_ = self.model.best_score_
self.feature_importances_ = self.model.feature_importances_
def predict_proba(self, X_test):
if self.model.objective == 'binary':
return self.model.predict_proba(X_test, num_iteration=self.model.best_iteration_)[:, 1]
else:
return self.model.predict_proba(X_test, num_iteration=self.model.best_iteration_ )<train_model>
data["PoolQC"] = data["PoolQC"].fillna("None")
 data["MiscFeature"] = data["MiscFeature"].fillna("None")
 data["Alley"] = data["Alley"].fillna("None")
 data["Fence"] = data["Fence"].fillna("None")
 data["FireplaceQu"] = data["FireplaceQu"].fillna("None")
 data["LotFrontage"] = data.groupby("Neighborhood")["LotFrontage"].transform(
 lambda x: x.fillna(x.median()))
for col in('GarageType', 'GarageFinish', 'GarageQual', 'GarageCond'):
data[col] = data[col].fillna('None')
 for col in('GarageYrBlt', 'GarageArea', 'GarageCars'):
data[col] = data[col].fillna(0)
 for col in('BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF','TotalBsmtSF', 'BsmtFullBath', 'BsmtHalfBath'):
data[col] = data[col].fillna(0)
 for col in('BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2'):
data[col] = data[col].fillna('None')
 data["MasVnrType"] = data["MasVnrType"].fillna("None")
 data["MasVnrArea"] = data["MasVnrArea"].fillna(0)
 data['MSZoning'] = data['MSZoning'].fillna(data['MSZoning'].mode() [0])
 data = data.drop(['Utilities'], axis=1)
 data["Functional"] = data["Functional"].fillna("Typ")
 data['Electrical'] = data['Electrical'].fillna(data['Electrical'].mode() [0])
 data['KitchenQual'] = data['KitchenQual'].fillna(data['KitchenQual'].mode() [0])
 data['Exterior1st'] = data['Exterior1st'].fillna(data['Exterior1st'].mode() [0])
 data['Exterior2nd'] = data['Exterior2nd'].fillna(data['Exterior2nd'].mode() [0])
 data['SaleType'] = data['SaleType'].fillna(data['SaleType'].mode() [0])
 data['MSSubClass'] = data['MSSubClass'].fillna("None" )
class RegressorModel(object):

 def __init__(self, columns: list = None, model_wrapper=None):

 self.columns = columns
self.model_wrapper = model_wrapper
self.result_dict = {}
self.train_one_fold = False
self.preprocesser = None
def fit(self, X: pd.DataFrame, y,
X_holdout: pd.DataFrame = None, y_holdout=None,
folds=None,
params: dict = None,
eval_metric='rmse',
cols_to_drop: list = None,
preprocesser=None,
transformers: dict = None,
adversarial: bool = False,
plot: bool = True):

 if folds is None:
folds = KFold(n_splits=3, random_state=42)
 self.train_one_fold = True
self.columns = X.columns if self.columns is None else self.columns
self.feature_importances = pd.DataFrame(columns=['feature', 'importance'])
 self.trained_transformers = {k: [] for k in transformers}
self.transformers = transformers
self.models = []
self.folds_dict = {}
self.eval_metric = eval_metric
n_target = 1
self.oof = np.zeros(( len(X), n_target))
self.n_target = n_target
X = X[self.columns]
if X_holdout is not None:
X_holdout = X_holdout[self.columns]
if preprocesser is not None:
self.preprocesser = preprocesser
self.preprocesser.fit(X, y)
 X = self.preprocesser.transform(X, y)
 self.columns = X.columns.tolist()
if X_holdout is not None:
X_holdout = self.preprocesser.transform(X_holdout)
 for fold_n,(train_index, valid_index)in enumerate(folds.split(X, y, X['installation_id'])) :
if X_holdout is not None:
X_hold = X_holdout.copy()
else:
X_hold = None
self.folds_dict[fold_n] = {}
if params['verbose']:
print(f'Fold {fold_n + 1} started at {time.ctime() }')
 self.folds_dict[fold_n] = {}
X_train, X_valid = X.iloc[train_index], X.iloc[valid_index]
y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]
if self.train_one_fold:
X_train = X[self.original_columns]
y_train = y
X_valid = None
y_valid = None
datasets = {'X_train': X_train, 'X_valid': X_valid, 'X_holdout': X_hold, 'y_train': y_train}
X_train, X_valid, X_hold = self.transform_(datasets, cols_to_drop)
 self.folds_dict[fold_n]['columns'] = X_train.columns.tolist()
model = copy.deepcopy(self.model_wrapper)
 if adversarial:
X_new1 = X_train.copy()
if X_valid is not None:
X_new2 = X_valid.copy()
elif X_holdout is not None:
X_new2 = X_holdout.copy()
X_new = pd.concat([X_new1, X_new2], axis=0)
 y_new = np.hstack(( np.zeros(( X_new1.shape[0])) , np.ones(( X_new2.shape[0]))))
X_train, X_valid, y_train, y_valid = train_test_split(X_new, y_new)
 model.fit(X_train, y_train, X_valid, y_valid, X_hold, y_holdout, params=params)
 self.folds_dict[fold_n]['scores'] = model.best_score_
if self.oof.shape[0] != len(X):
self.oof = np.zeros(( X.shape[0], self.oof.shape[1]))
if not adversarial:
self.oof[valid_index] = model.predict(X_valid ).reshape(-1, n_target)
 fold_importance = pd.DataFrame(list(zip(X_train.columns, model.feature_importances_)) ,
columns=['feature', 'importance'])
 self.feature_importances = self.feature_importances.append(fold_importance)
 self.models.append(model)
 self.feature_importances['importance'] = self.feature_importances['importance'].astype(int)
 self.calc_scores_()
if plot:
fig, ax = plt.subplots(figsize=(16, 12))
plt.subplot(2, 2, 1)
 self.plot_feature_importance(top_n=20)
 plt.subplot(2, 2, 2)
 self.plot_metric()
plt.subplot(2, 2, 3)
 plt.hist(y.values.reshape(-1, 1)- self.oof)
 plt.title('Distribution of errors')
 plt.subplot(2, 2, 4)
 plt.hist(self.oof)
 plt.title('Distribution of oof predictions');
def transform_(self, datasets, cols_to_drop):
for name, transformer in self.transformers.items() :
transformer.fit(datasets['X_train'], datasets['y_train'])
 datasets['X_train'] = transformer.transform(datasets['X_train'])
 if datasets['X_valid'] is not None:
datasets['X_valid'] = transformer.transform(datasets['X_valid'])
 if datasets['X_holdout'] is not None:
datasets['X_holdout'] = transformer.transform(datasets['X_holdout'])
 self.trained_transformers[name].append(transformer)
 if cols_to_drop is not None:
cols_to_drop = [col for col in cols_to_drop if col in datasets['X_train'].columns]
datasets['X_train'] = datasets['X_train'].drop(cols_to_drop, axis=1)
 if datasets['X_valid'] is not None:
datasets['X_valid'] = datasets['X_valid'].drop(cols_to_drop, axis=1)
 if datasets['X_holdout'] is not None:
datasets['X_holdout'] = datasets['X_holdout'].drop(cols_to_drop, axis=1)
 self.cols_to_drop = cols_to_drop
return datasets['X_train'], datasets['X_valid'], datasets['X_holdout']
def calc_scores_(self):
datasets = [k for k, v in [v['scores'] for k, v in self.folds_dict.items() ][0].items() if len(v)> 0]
self.scores = {}
for d in datasets:
scores = [v['scores'][d][self.eval_metric] for k, v in self.folds_dict.items() ]
self.scores[d] = np.mean(scores)
 def predict(self, X_test, averaging: str = 'usual'):

 full_prediction = np.zeros(( X_test.shape[0], self.oof.shape[1]))
if self.preprocesser is not None:
X_test = self.preprocesser.transform(X_test)
 for i in range(len(self.models)) :
X_t = X_test.copy()
for name, transformers in self.trained_transformers.items() :
X_t = transformers[i].transform(X_t)
 if self.cols_to_drop is not None:
cols_to_drop = [col for col in self.cols_to_drop if col in X_t.columns]
X_t = X_t.drop(cols_to_drop, axis=1)
 y_pred = self.models[i].predict(X_t[self.folds_dict[i]['columns']] ).reshape(-1, full_prediction.shape[1])
 if full_prediction.shape[0] != len(y_pred):
full_prediction = np.zeros(( y_pred.shape[0], self.oof.shape[1]))
if averaging == 'usual':
full_prediction += y_pred
elif averaging == 'rank':
full_prediction += pd.Series(y_pred ).rank().values
return full_prediction / len(self.models)
 def plot_feature_importance(self, drop_null_importance: bool = True, top_n: int = 10):

 top_feats = self.get_top_features(drop_null_importance, top_n)
 feature_importances = self.feature_importances.loc[self.feature_importances['feature'].isin(top_feats)]
feature_importances['feature'] = feature_importances['feature'].astype(str)
 top_feats = [str(i)for i in top_feats]
sns.barplot(data=feature_importances, x='importance', y='feature', orient='h', order=top_feats)
 plt.title('Feature importances')
 def get_top_features(self, drop_null_importance: bool = True, top_n: int = 10):

 grouped_feats = self.feature_importances.groupby(['feature'])['importance'].mean()
if drop_null_importance:
grouped_feats = grouped_feats[grouped_feats != 0]
return list(grouped_feats.sort_values(ascending=False ).index)[:top_n]
def plot_metric(self):

 full_evals_results = pd.DataFrame()
for model in self.models:
evals_result = pd.DataFrame()
for k in model.model.evals_result_.keys() :
evals_result[k] = model.model.evals_result_[k][self.eval_metric]
evals_result = evals_result.reset_index().rename(columns={'index': 'iteration'})
 full_evals_results = full_evals_results.append(evals_result)
 full_evals_results = full_evals_results.melt(id_vars=['iteration'] ).rename(columns={'value': self.eval_metric,
'variable': 'dataset'})
 sns.lineplot(data=full_evals_results, x='iteration', y=self.eval_metric, hue='dataset')
 plt.title('Training progress' )<prepare_x_and_y>
data['MSSubClass'] = data['MSSubClass'].apply(str)
 data['OverallCond'] = data['OverallCond'].astype(str)
 data['YrSold'] = data['YrSold'].astype(str)
 data['MoSold'] = data['MoSold'].astype(str )
y = reduce_train['accuracy_group']
cols_to_drop = ['game_session', 'installation_id', 'timestamp', 'accuracy_group', 'timestampDate']
n_fold = 6
folds = GroupKFold(n_splits=n_fold )<choose_model_class>
cols =('FireplaceQu', 'BsmtQual', 'BsmtCond', 'GarageQual', 'GarageCond',
'ExterQual', 'ExterCond','HeatingQC', 'PoolQC', 'KitchenQual', 'BsmtFinType1',
'BsmtFinType2', 'Functional', 'Fence', 'BsmtExposure', 'GarageFinish', 'LandSlope',
'LotShape', 'PavedDrive', 'Street', 'Alley', 'CentralAir', 'MSSubClass', 'OverallCond',
'YrSold', 'MoSold')
 for c in cols:
lbl = LabelEncoder()
lbl.fit(list(data[c].values))
data[c] = lbl.transform(list(data[c].values))
print('Shape data: {}'.format(data.shape))
def LGB_bayesian(max_depth,
lambda_l1,
lambda_l2,
bagging_fraction,
bagging_freq,
colsample_bytree,
learning_rate):
params = {
'boosting_type': 'gbdt',
'metric': 'rmse',
'objective': 'regression',
'eval_metric': 'cappa',
'n_jobs': -1,
'seed': 42,
'early_stopping_rounds': 100,
'n_estimators': 2000,
'learning_rate': learning_rate,
'max_depth': int(max_depth),
'lambda_l1': lambda_l1,
'lambda_l2': lambda_l2,
'bagging_fraction': bagging_fraction,
'bagging_freq': int(bagging_freq),
'colsample_bytree': colsample_bytree,
'verbose': 0
}
mt = MainTransformer()
ft = FeatureTransformer()
transformers = {'ft': ft}
model = RegressorModel(model_wrapper=LGBWrapper_regr())
 model.fit(X=reduce_train,
y=y,
folds=folds,
params=params,
preprocesser=mt,
transformers=transformers,
eval_metric='cappa',
cols_to_drop=cols_to_drop,
plot=False)
 return model.scores['valid']<set_options>
print("There are {} skewed numerical features to Box Cox transform".format(skewness.shape[0]))
skewed_features = skewness.index
lam = 0.15
for feat in skewed_features:
data[feat] = boxcox1p(data[feat], lam)
gc.collect()<choose_model_class>
data_na =(data.isnull().sum() / len(data)) * 100
data_na = data_na.drop(data_na[data_na == 0].index ).sort_values(ascending=False)
 missing_data = pd.DataFrame({'Missing Ratio' :data_na})
 missing_data.head()
init_points = 16
n_iter = 16
bounds_LGB = {
'max_depth':(8, 12),
'lambda_l1':(0, 5),
'lambda_l2':(0, 5),
'bagging_fraction':(0.4, 0.6),
'bagging_freq':(1, 10),
'colsample_bytree':(0.4, 0.6),
'learning_rate':(0.025, 0.1)
 }
LGB_BO = BayesianOptimization(LGB_bayesian, bounds_LGB, random_state=1029)
 with warnings.catch_warnings() :
warnings.filterwarnings('ignore')
 LGB_BO.maximize(init_points=init_points, n_iter=n_iter, acq='ucb', xi=0.0, alpha=1e-6)
 print(LGB_BO.max )<init_hyperparams>
data['TotalSF'] = data['TotalBsmtSF'] + data['1stFlrSF'] + data['2ndFlrSF']
params = {
'boosting_type': 'gbdt',
'metric': 'rmse',
'objective': 'regression',
'eval_metric': 'cappa',
'n_jobs': -1,
'seed': 42,
'early_stopping_rounds': 100,
'n_estimators': 2000,
'learning_rate': LGB_BO.max['params']['learning_rate'],
'max_depth': int(LGB_BO.max['params']['max_depth']),
'lambda_l1': LGB_BO.max['params']['lambda_l1'],
'lambda_l2': LGB_BO.max['params']['lambda_l2'],
'bagging_fraction': LGB_BO.max['params']['bagging_fraction'],
'bagging_freq': int(LGB_BO.max['params']['bagging_freq']),
'colsample_bytree': LGB_BO.max['params']['colsample_bytree'],
'verbose': 100
}
mt = MainTransformer()
ft = FeatureTransformer()
transformers = {'ft': ft}
regressor_model = RegressorModel(model_wrapper=LGBWrapper_regr())
 regressor_model.fit(X=reduce_train,
y=y,
folds=folds,
params=params,
preprocesser=mt,
transformers=transformers,
eval_metric='cappa',
cols_to_drop=cols_to_drop)
 preds_1 = regressor_model.predict(reduce_test)

w_1 = LGB_BO.max['target']<drop_column>
data = pd.get_dummies(data)
 print(data.shape )
del bounds_LGB, LGB_BO, params, mt, ft, transformers, regressor_model
gc.collect()<find_best_params>
train = data[:ntrain]
test = data[ntrain:]
bounds_LGB = {
'max_depth':(10, 14),
'lambda_l1':(0, 10),
'lambda_l2':(0, 10),
'bagging_fraction':(0.7, 1),
'bagging_freq':(1, 10),
'colsample_bytree':(0.7, 1),
'learning_rate':(0.06, 0.24)
 }
LGB_BO = BayesianOptimization(LGB_bayesian, bounds_LGB, random_state=1030)
 with warnings.catch_warnings() :
warnings.filterwarnings('ignore')
 LGB_BO.maximize(init_points=init_points, n_iter=n_iter, acq='ucb', xi=0.0, alpha=1e-6)
 print(LGB_BO.max )<init_hyperparams>
from sklearn.linear_model import ElasticNet, Lasso, BayesianRidge, LassoLarsIC
from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor
from sklearn.kernel_ridge import KernelRidge
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import RobustScaler
from sklearn.base import BaseEstimator, TransformerMixin, RegressorMixin, clone
from sklearn.model_selection import KFold, cross_val_score, train_test_split
from sklearn.metrics import mean_squared_error
import xgboost as xgb
import lightgbm as lgb
params = {
'boosting_type': 'gbdt',
'metric': 'rmse',
'objective': 'regression',
'eval_metric': 'cappa',
'n_jobs': -1,
'seed': 42,
'early_stopping_rounds': 100,
'n_estimators': 2000,
'learning_rate': LGB_BO.max['params']['learning_rate'],
'max_depth': int(LGB_BO.max['params']['max_depth']),
'lambda_l1': LGB_BO.max['params']['lambda_l1'],
'lambda_l2': LGB_BO.max['params']['lambda_l2'],
'bagging_fraction': LGB_BO.max['params']['bagging_fraction'],
'bagging_freq': int(LGB_BO.max['params']['bagging_freq']),
'colsample_bytree': LGB_BO.max['params']['colsample_bytree'],
'verbose': 100
}
mt = MainTransformer()
ft = FeatureTransformer()
transformers = {'ft': ft}
regressor_model = RegressorModel(model_wrapper=LGBWrapper_regr())
 regressor_model.fit(X=reduce_train,
y=y,
folds=folds,
params=params,
preprocesser=mt,
transformers=transformers,
eval_metric='cappa',
cols_to_drop=cols_to_drop)
 preds_2 = regressor_model.predict(reduce_test)
 w_2 = LGB_BO.max['target']<drop_column>
n_folds = 5
def rmsle_cv(model):
kf = KFold(n_folds, shuffle=True, random_state=42 ).get_n_splits(train.values)
 rmse= np.sqrt(-cross_val_score(model, train.values, y_train, scoring="neg_mean_squared_error", cv = kf))
return(rmse )
del bounds_LGB, LGB_BO, params, mt, ft, transformers, regressor_model
gc.collect()<drop_column>
lasso = make_pipeline(RobustScaler() , Lasso(alpha =0.0005, random_state=1))
preds =(w_1/(w_1+w_2)) * preds_1 +(w_2/(w_1+w_2)) * preds_2
del preds_1, preds_2
gc.collect()<feature_engineering>
score = rmsle_cv(lasso)
 print("
Lasso score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
coefficients = [1.12232214, 1.73925866, 2.22506454]
preds[preds <= coefficients[0]] = 0
preds[np.where(np.logical_and(preds > coefficients[0], preds <= coefficients[1])) ] = 1
preds[np.where(np.logical_and(preds > coefficients[1], preds <= coefficients[2])) ] = 2
preds[preds > coefficients[2]] = 3<data_type_conversions>
ENet = make_pipeline(RobustScaler() , ElasticNet(alpha=0.0005, l1_ratio=.9, random_state=3))
sample_submission['accuracy_group'] = preds.astype(int)
 sample_submission['accuracy_group'].value_counts(normalize=True )<save_to_csv>
score = rmsle_cv(ENet)
 print("ElasticNet score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
sample_submission.to_csv('submission.csv', index=False )<import_modules>
KRR = KernelRidge(alpha=0.8, kernel='polynomial', degree=2, coef0=3.5 )
import os
import pandas as pd
import seaborn as sns
import numpy as np
import datetime
import sklearn
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LogisticRegressionCV
from sklearn import model_selection
from sklearn.model_selection import cross_val_score
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import confusion_matrix,classification_report
import matplotlib.pyplot as plt
from pprint import pprint
from bayes_opt import BayesianOptimization
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import cohen_kappa_score
import xgboost as xgb
from xgboost import XGBClassifier
from xgboost import plot_importance
from sklearn.model_selection import StratifiedKFold, GroupKFold,GroupShuffleSplit,StratifiedShuffleSplit
from sklearn.preprocessing import LabelEncoder
from sklearn import model_selection
from sklearn.cross_decomposition import PLSRegression, PLSSVD
from sklearn.metrics import mean_squared_error
from sklearn.preprocessing import scale
from sklearn.utils import class_weight
from sklearn.preprocessing import StandardScaler
from sklearn.decomposition import PCA
import hashlib
from ml_metrics import quadratic_weighted_kappa
from catboost import CatBoostClassifier,Pool,CatBoostRegressor
import shap
import statistics
from functools import partial
import scipy as sp
from lightgbm import LGBMRegressor
import lightgbm as lgb
from xgboost import XGBRegressor
from sklearn.ensemble import RandomForestRegressor
from sklearn.ensemble import AdaBoostRegressor
from mlxtend.regressor import StackingCVRegressor
import pickle
from sklearn.model_selection import cross_val_score
from sklearn.utils import shuffle<load_from_csv>
score = rmsle_cv(KRR)
 print("Kernel Ridge score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
df_train=pd.read_csv('/kaggle/input/data-science-bowl-2019/train.csv' )<load_from_csv>
GBoost = GradientBoostingRegressor(n_estimators=1000, learning_rate=0.05,
max_depth=2, max_features='sqrt',
min_samples_leaf=15, min_samples_split=10,
loss='huber', random_state=42 )
spec=pd.read_csv('/kaggle/input/data-science-bowl-2019/specs.csv' )<load_from_csv>
score = rmsle_cv(GBoost)
 print("Gradient Boosting score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
df_test=pd.read_csv('/kaggle/input/data-science-bowl-2019/test.csv' )<merge>
model_xgb = xgb.XGBRegressor(colsample_bytree=0.5604, gamma=0.0578,
learning_rate=0.05, max_depth=2,
min_child_weight=1.7817, n_estimators=900,
reg_alpha=0.4765, reg_lambda=0.9173,
subsample=0.4738, silent=1,
random_state =42, nthread = -1 )
spec['info']=spec['info'].str.upper()
spec['hashed_info']=spec['info'].transform(hash)
 spec_unique=pd.DataFrame(spec[['hashed_info']].drop_duplicates())
 spec_unique['deduped_event_id']=np.arange(len(spec_unique))
spec=pd.merge(spec,spec_unique,on='hashed_info',how='left')
 z=dict(zip(spec.event_id,spec.deduped_event_id))
df_train['event_id']=df_train['event_id'].map(z)
 df_test['event_id']=df_test['event_id'].map(z)
 df_train=df_train[df_train['event_id']!=137]
df_event_id_train=pd.pivot_table(df_train.loc[:,['installation_id','game_session','event_id']],aggfunc=len,columns=['event_id'],index=['installation_id','game_session'] ).add_prefix('event_id_' ).rename_axis(None,axis=1 ).reset_index()
df_event_id_test=pd.pivot_table(df_test.loc[:,['installation_id','game_session','event_id']],aggfunc=len,columns=['event_id'],index=['installation_id','game_session'] ).add_prefix('event_id_' ).rename_axis(None,axis=1 ).reset_index()
df_event_id_train=df_event_id_train.fillna(0)
 df_event_id_train=df_event_id_train.fillna(0)
 df_event_id_test=df_event_id_test.fillna(0 )<feature_engineering>
score = rmsle_cv(model_xgb)
 print("Xgboost score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
def create_features(df):
df['timestamp']=pd.to_datetime(df['timestamp'])
 df['Incorrect_Game_Attempt']=np.where(( df['event_data'].str.contains('"correct":false')&(df['type']=='Game')) ,1,0)
 df['Correct_Game_Attempt']=np.where(( df['event_data'].str.contains('"correct":true')&(df['type']=='Game')) ,1,0)
 df['Is_Weekend']=np.where(((df['timestamp'].dt.day_name() =='Sunday')|(df['timestamp'].dt.day_name() =='Saturday')) ,1,0)
 df['Phase_Of_Day']=np.where(df['timestamp'].dt.hour.isin(range(6,12)) ,'Morning',np.where(df['timestamp'].dt.hour.isin(range(13,19)) ,'Evening','Night'))
df_world=pd.pivot_table(df.loc[df['world']!='NONE',['installation_id','game_session','world']].drop_duplicates() ,index=['installation_id','game_session'],columns=['world'],aggfunc=len ).add_prefix('rolling_' ).rename_axis(None, axis=1 ).reset_index()
df_type_world=pd.merge(df_world,pd.pivot_table(df.loc[:,['installation_id','game_session','type']].drop_duplicates() ,index=['installation_id','game_session'],columns=['type'],fill_value=0,aggfunc=len ).rename_axis(None, axis=1 ).reset_index() ,on=['installation_id','game_session'],how='right')
 df_type_world_title=pd.merge(df_type_world,pd.pivot_table(df.loc[:,['installation_id','game_session','title']].drop_duplicates() ,index=['installation_id','game_session'],columns=['title'],fill_value=0,aggfunc=len ).add_prefix('rolling_' ).rename_axis(None, axis=1 ).reset_index() ,on=['installation_id','game_session'],how='right')
 df_activity_weekend=pd.merge(df_type_world_title,pd.DataFrame(pd.pivot_table(df.loc[:,['installation_id','game_session','Is_Weekend']].drop_duplicates() ,index=['installation_id','game_session'],columns=['Is_Weekend'],fill_value=0,aggfunc=len)).add_prefix('Weekend_' ).rename_axis(None, axis=1 ).reset_index() ,on=['installation_id','game_session'],how='right')
 df_activity_weekend_phase_of_day=pd.merge(pd.DataFrame(pd.pivot_table(df.loc[:,['installation_id','game_session','Phase_Of_Day']].drop_duplicates() ,index=['installation_id','game_session'],columns=['Phase_Of_Day'],fill_value=0,aggfunc=len)).rename_axis(None, axis=1 ).reset_index() ,df_activity_weekend,on=['installation_id','game_session'],how='left')
 df_train_Assessed=df.copy()
df_train_Assessed['Incorrect_Attempt']=np.where(( df['event_data'].str.contains('"correct":false')) &(((df['title'] != "Bird Measurer(Assessment)")&(df['event_code']==4100)) |(( df['title'] == "Bird Measurer(Assessment)")&(df['event_code']==4110))),1,0)
 df_train_Assessed['Correct_Attempt']=np.where(( df['event_data'].str.contains('"correct":true')) &(((df['title'] != "Bird Measurer(Assessment)")&(df['event_code']==4100)) |(( df['title'] == "Bird Measurer(Assessment)")&(df['event_code']==4110))),1,0)
 df_train_acc=df_train_Assessed[df_train_Assessed['title'].isin(['Bird Measurer(Assessment)','Mushroom Sorter(Assessment)','Cauldron Filler(Assessment)','Chest Sorter(Assessment)','Cart Balancer(Assessment)'])].groupby(['installation_id','title','game_session'])['Incorrect_Attempt','Correct_Attempt'].sum().rename_axis(None, axis=1 ).reset_index()
df_train_acc['Total_Attempts']=df_train_acc.apply(lambda x: x['Incorrect_Attempt'] + x['Correct_Attempt'], axis=1)
 df_train_acc['accuracy']=np.where(df_train_acc['Total_Attempts']>0,df_train_acc['Correct_Attempt']/ df_train_acc['Total_Attempts'],0)
 df_train_acc['accuracy_group']=np.where(df_train_acc['accuracy']==1,3,np.where(df_train_acc['accuracy']==.5,2,np.where(df_train_acc['accuracy']==0,0,1)))
 df_game_attempt=df.groupby(['installation_id','game_session'])['Incorrect_Game_Attempt','Correct_Game_Attempt'].sum().rename_axis(None, axis=1 ).reset_index()
df_event_codes=pd.pivot_table(df_train_Assessed.loc[:,['installation_id','game_session','event_code']],index=['installation_id','game_session'],columns=['event_code'],fill_value=0,aggfunc=len ).add_prefix('event_code_' ).rename_axis(None, axis=1 ).reset_index()
df_final=pd.merge(pd.merge(df_train_acc,df_activity_weekend_phase_of_day,on=['installation_id','game_session'],how='right'),df_event_codes,on=['installation_id','game_session'],how='right')
 df_gametime=df.groupby(['installation_id','game_session'])['game_time','timestamp','event_count'].max().reset_index()
df_final=pd.merge(df_final,df_gametime,on=['installation_id','game_session'],how='left')
 df_final=df_final.fillna(value=0)
 df_final=pd.merge(df_final,df.loc[df['world']!="NONE",['installation_id','game_session','world']].drop_duplicates() ,on=['installation_id','game_session'],how='left')
 df_final=pd.merge(df_final,df_game_attempt,on=['installation_id','game_session'],how='left')
 df_final=df_final.fillna(value=0)
 return(df_final )<groupby>
model_lgb = lgb.LGBMRegressor(objective='regression',num_leaves=5,
learning_rate=0.05, n_estimators=720,
max_bin = 55, bagging_fraction = 0.8,
bagging_freq = 5, feature_fraction = 0.2319,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf =6, min_sum_hessian_in_leaf = 11 )
def rolling_exponential_average(df):
col=list(df.select_dtypes(include=['float64','int64'] ).columns)
 df=df.sort_values(['installation_id','timestamp'])
 df_rolling_avg=df.groupby(['installation_id'])[col].apply(lambda x:x.ewm(alpha=0.1,min_periods=1 ).mean())
 df_rolling_avg=df_rolling_avg.rename(columns={'accuracy_group':'rolling_accuracy_group','CRYSTALCAVES':'rolling_CRYSTALCAVES','MAGMAPEAK':'rolling_MAGMAPEAK', 'TREETOPCITY':'rolling_TREETOPCITY'})
 df_rolling_avg.index.names=['level_1']
return(df_rolling_avg )<merge>
score = rmsle_cv(model_lgb)
 print("LGBM score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
def create_final_dataset(df,Is_Test=0):
f=create_features(df)
 df_assessed=set(df.loc[df['type']=='Assessment'] 
.loc[(( df['title'] != "Bird Measurer(Assessment)")&(df['event_code']==4100)) | 
(( df['title'] == "Bird Measurer(Assessment)")&(df['event_code']==4110)) ] 
.loc[:,'game_session'].unique())
 if(Is_Test==0):
f=pd.merge(f,df_event_id_train,on=['installation_id','game_session'],how='left')
 else:
f=pd.merge(f,df_event_id_test,on=['installation_id','game_session'],how='left')
 df_rolling_avg=rolling_exponential_average(f)
 df_rolling_avg=df_rolling_avg.fillna(0)
 f.index.names=['level_1']
t=pd.merge(f[['installation_id','game_session','timestamp','world','title']],df_rolling_avg,left_index=True,right_index=True,how='left')
 t=t.reset_index(drop=True ).sort_values(['installation_id','timestamp'])
 tm=pd.merge(t[['installation_id','game_session','title','world']],t.groupby(['installation_id'] ).shift(1 ).drop(['game_session','title','world'],axis=1),left_index=True,right_index=True,how='left')
 tm=tm.dropna()
tm=tm[tm['game_session'].isin(df_assessed)]
if(Is_Test==0):
tm['last_attempt_time']=tm.groupby(['installation_id'] ).shift(1)['timestamp']
tm['last_world']=tm.groupby(['installation_id'] ).shift(1)['world']
tm['last_title']=tm.groupby(['installation_id'] ).shift(1)['title']
tm['time_since_last_attempt']=np.where(tm['last_attempt_time'].isna() ,0,(tm['timestamp']-tm['last_attempt_time'] ).dt.seconds)
 else:
df_latest_Attempt=t.sort_values(['installation_id','timestamp'] ).groupby(['installation_id'],as_index=False ).last()
tm=tm.sort_values(['installation_id','timestamp','title','world'] ).groupby(['installation_id'],as_index=False ).last().rename(columns={'timestamp':'last_attempt_time','world':'last_world','title':'last_title'} ).drop_duplicates()
df_latest_Attempt=pd.merge(tm[['installation_id','last_attempt_time','last_world','last_title']],df_latest_Attempt,on=['installation_id'],how='right')
 df_latest_Attempt['time_since_last_attempt']=np.where(df_latest_Attempt['last_attempt_time'].isna() ,0,(pd.to_datetime(df_latest_Attempt['timestamp'].dt.tz_localize(None)) -pd.to_datetime(df_latest_Attempt['last_attempt_time'])).dt.seconds)
 tm=df_latest_Attempt.copy()
tm1=pd.merge(tm,f[['accuracy_group','game_session','installation_id']],on=['game_session','installation_id'],how='left')
 tm1=tm1.fillna(0)
 return(tm1 )<create_dataframe>
class AveragingModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, models):
self.models = models
def fit(self, X, y):
self.models_ = [clone(x)for x in self.models]
for model in self.models_:
model.fit(X, y)
 return self
def predict(self, X):
predictions = np.column_stack([
model.predict(X)for model in self.models_
])
 return np.mean(predictions, axis=1 )
f_train=create_final_dataset(df_train,0)
 f_test=create_final_dataset(df_test,1)
 f_train = f_train.reindex(sorted(f_train.columns), axis=1)
 f_test = f_test.reindex(sorted(f_test.columns), axis=1 )<normalization>
averaged_models = AveragingModels(models =(ENet, GBoost, KRR, lasso))
score = rmsle_cv(averaged_models)
 print(" Averaged base models score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
f_train_1=shuffle(f_train )<drop_column>
class StackingAveragedModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, base_models, meta_model, n_folds=5):
self.base_models = base_models
self.meta_model = meta_model
self.n_folds = n_folds
def fit(self, X, y):
self.base_models_ = [list() for x in self.base_models]
self.meta_model_ = clone(self.meta_model)
 kfold = KFold(n_splits=self.n_folds, shuffle=True, random_state=156)
 out_of_fold_predictions = np.zeros(( X.shape[0], len(self.base_models)))
 for i, model in enumerate(self.base_models):
for train_index, holdout_index in kfold.split(X, y):
instance = clone(model)
 self.base_models_[i].append(instance)
 instance.fit(X[train_index], y[train_index])
 y_pred = instance.predict(X[holdout_index])
 out_of_fold_predictions[holdout_index, i] = y_pred
self.meta_model_.fit(out_of_fold_predictions, y)
 return self
def predict(self, X):
meta_features = np.column_stack([
np.column_stack([model.predict(X)for model in base_models] ).mean(axis=1)
 for base_models in self.base_models_ ])
 return self.meta_model_.predict(meta_features )
f_train_1.reset_index(inplace=True, drop=True )<train_on_grid>
stacked_averaged_models = StackingAveragedModels(base_models =(ENet, GBoost, KRR),
meta_model = lasso)
 score = rmsle_cv(stacked_averaged_models)
 print("Stacking Averaged models score: {:.4f}({:.4f})".format(score.mean() , score.std()))
class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
preds = pd.cut(X, [-np.inf] + list(np.sort(coef)) + [np.inf], labels = [0, 1, 2, 3])
 return -cohen_kappa_score(y, preds, weights = 'quadratic')
 def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X = X, y = y)
 initial_coef = [0.5, 1.5, 2.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method = 'nelder-mead')
 def predict(self, X, coef):
preds = pd.cut(X, [-np.inf] + list(np.sort(coef)) + [np.inf], labels = [0, 1, 2, 3])
 return preds
def coefficients(self):
return self.coef_['x']<train_model>
def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))
def model_classification(train_X,train_y,val_X,val_y,cat_feat):
CatBoost=CatBoostClassifier(
 learning_rate=0.2,
max_depth=6,
bootstrap_type='Bernoulli',
l2_leaf_reg= 0.2,
iterations=400,
subsample=.7,
loss_function='MultiClass',
eval_metric="WKappa",
random_seed=4,cat_features=cat_feat,class_weights=[1,2,2,.5],
classes_count=4)
 clf = CatBoost.fit(train_X,train_y,eval_set=(val_X,val_y), verbose_eval=False,use_best_model=True,plot=True,early_stopping_rounds=100)
 return(clf)
<train_model>
stacked_averaged_models.fit(train.values, y_train)
 stacked_train_pred = stacked_averaged_models.predict(train.values)
 stacked_pred = np.expm1(stacked_averaged_models.predict(test.values))
print(rmsle(y_train, stacked_train_pred))
def model_regressor(train_X,train_y,val_X,val_y,fit):
lgb_model=LGBMRegressor(objective='regression',
max_depth=13,num_leaves=6,
n_estimators=1200,
bagging_fraction=0.8,
random_state=42,verbose=0,reg_alpha=.3,reg_lambda=.3,callbacks=[lgb.reset_parameter(learning_rate=[0.2] * 200 + [0.15] * 250+[.1]*250+[.05]*250+[.01]*250)])
 if fit==0:
reg=lgb_model.fit(train_X,train_y,eval_set=(val_X,val_y), verbose=False)
 else:
reg=lgb_model.fit(train_X,train_y,verbose=False)
 return(reg)
<compute_train_metric>
model_xgb.fit(train, y_train)
 xgb_train_pred = model_xgb.predict(train)
 xgb_pred = np.expm1(model_xgb.predict(test))
print(rmsle(y_train, xgb_train_pred))
def fit(X_train,y_train,classifier):
train_X = X_train
train_y=y_train
train_X=train_X.drop('installation_id',axis=1)
 if(classifier==1):
clf=model_classification(train_X,train_y,0,0,cat_feat)
 pred_val=clf.predict(val_X)
 print(classification_report(val_y,pred_val))
score=cohen_kappa_score(pred_val,val_y,weights='quadratic')
 scores.append(score)
 print('choen_kappa_score :',score)
 coeff=0
else:
clf=model_regressor(train_X,train_y,0,0,1)
 optR = OptimizedRounder()
optR.fit(clf.predict(train_X), train_y)
 coeff = optR.coefficients()
return(clf,coeff )<train_on_grid>
model_lgb.fit(train, y_train)
 lgb_train_pred = model_lgb.predict(train)
 lgb_pred = np.expm1(model_lgb.predict(test.values))
print(rmsle(y_train, lgb_train_pred))
def cv(X_train,y_train,classifier=1,n_splits=5):
scores=[]
Gf=GroupKFold(n_splits=n_splits)
 i=0
for train_index, val_index in Gf.split(X_train, y_train,X_train['installation_id']):
train_X = X_train.iloc[train_index]
train_X=train_X.drop('installation_id',axis=1)
 val=X_train.iloc[val_index].join(y_train[val_index])
 train_y = y_train[train_index]
val=val.groupby('installation_id',as_index=False ).apply(lambda x :x.iloc[np.random.randint(0,len(x)) ])
 val_X=val.drop(['accuracy_group','installation_id'],axis=1)
 val_y=val['accuracy_group']
if(classifier==1):
clf=model_classification(train_X,train_y,val_X,val_y,0)
 pred_val=clf.predict(val_X)
 print(classification_report(val_y,pred_val))
score=cohen_kappa_score(pred_val,val_y,weights='quadratic')
 scores.append(score)
 print('choen_kappa_score :',score)
 else:
clf=model_regressor(train_X,train_y,val_X,val_y,0)
 print(clf.predict(train_X))
optR = OptimizedRounder()
optR.fit(clf.predict(train_X), train_y)
 coeff = optR.coefficients()
print(coeff)
 val_pred=clf.predict(val_X)
 pred_val=[regression_class(val_pred_i,coeff)for val_pred_i in val_pred]
print(classification_report(val_y,pred_val))
score=cohen_kappa_score(pred_val,val_y,weights='quadratic')
 scores.append(score)
 print('choen_kappa_score :',score)
 print(statistics.mean(scores),statistics.stdev(scores))
return(clf)
<compute_test_metric>

 print('RMSLE score on train data:')
 print(rmsle(y_train,stacked_train_pred*0.5 +
xgb_train_pred*0.1 + lgb_train_pred*0.4))
def regression_class(pred,coeff):
if(pred<=coeff[0]):
class_=0
elif(coeff[0]<pred<=coeff[1]):
class_=1
elif(coeff[1]<pred<=coeff[2]):
class_=2
else:
class_=3
return(class_ )<data_type_conversions>
ensemble = stacked_pred*0.5 + xgb_pred*0.1 + lgb_pred*0.4
<prepare_x_and_y><EOS> 
sub = pd.DataFrame()
sub['Id'] = test_ID
sub['SalePrice'] = ensemble
sub.to_csv('submission.csv',index=False )