Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<statistical_test>
!pip install bayesian-optimization
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix_1 = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix_1[i] = embedding_vector
del embeddings_index; gc.collect()<statistical_test>
color = sns.color_palette()
sns.set_style('darkgrid')
 def ignore_warn(*args, **kwargs):
pass
warnings.warn = ignore_warn
np.random.RandomState(seed=1)
EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix_2 = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix_2[i] = embedding_vector
del embeddings_index; gc.collect()<statistical_test>
train=pd.read_csv(".. /input/train.csv")
 test=pd.read_csv(".. /input/test.csv")
 train_Id=train.Id
test_Id=test.Id
train=train.drop(['Id'],axis=1)
 test=test.drop(['Id'],axis=1 )
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix_3 = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix_3[i] = embedding_vector
del embeddings_index; gc.collect()<concatenate>
train = train.drop(train[(train['GrLivArea']>4000)&(train['SalePrice']<300000)].index )
embedding_matrix = np.concatenate(( embedding_matrix_1, embedding_matrix_2, embedding_matrix_3), axis=1)
 del embedding_matrix_1, embedding_matrix_2, embedding_matrix_3
gc.collect()
np.shape(embedding_matrix )<split>
all_data=pd.concat([train,test])
 missing=(all_data.isnull().sum().sort_values(ascending=False)/len(all_data)) [:35]
missing_data = pd.DataFrame({'Missing Ratio' :missing})
 missing_data.head(35 )
X_tra, X_val, y_tra, y_val = train_test_split(x_train, y, test_size = 0.1, random_state=42 )<choose_model_class>
for cols in('MiscFeature','PoolQC','Alley','Fence','BsmtFinType1','KitchenQual','BsmtCond','BsmtQual','BsmtExposure','BsmtFinType2','GarageQual','GarageFinish','GarageCond','GarageType','FireplaceQu','MasVnrType','Exterior2nd'):
all_data[cols]=all_data[cols].fillna("None" )
<choose_model_class>
for cols in('GarageYrBlt','MasVnrArea','BsmtHalfBath','BsmtFullBath','GarageCars','GarageArea','TotalBsmtSF','BsmtUnfSF','BsmtFinSF2','BsmtFinSF1'):
all_data[cols]=all_data[cols].fillna(0 )
def model2() :
inp = Input(shape=(maxlen,))
embed = Embedding(max_features, embed_size * 3, weights=[embedding_matrix], trainable=False )(inp)
 x = embed
x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)
 x = attention_3d_block(x)
 x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)
 x = AttLayer(64 )(x)
 x = Dropout(0.3 )(x)
 x = Dense(128, activation='relu' )(x)
 outp = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=outp)
 model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
 return model<train_model>
for cols in('MSZoning','Electrical','Exterior2nd','Exterior1st','SaleType'):
all_data[cols]=all_data[cols].fillna(all_data[cols].mode() [0])
MODEL2 = model2()
MODEL2.summary()
batch_size = 2048
epochs = 3
early_stopping = EarlyStopping(patience=3, verbose=1, monitor='val_loss', mode='min')
 model_checkpoint = ModelCheckpoint('./model2.model', save_best_only=True, verbose=1, monitor='val_loss', mode='min')
 reduce_lr = ReduceLROnPlateau(factor=0.5, patience=3, min_lr=0.0001, verbose=1)
 hist = MODEL2.fit(X_tra, y_tra, batch_size=batch_size, epochs=epochs, validation_data=(X_val, y_val), verbose=True)
 MODEL2.save('./model2.h5' )<predict_on_test>
all_data=all_data.drop(['Utilities'],axis=1)
 all_data['Functional']=all_data['Functional'].fillna('Typ')
 all_data['LotFrontage']=all_data.groupby(['Neighborhood'] ).transform(
 lambda x: x.fillna(x.median())
)
pred_val_y_2 = MODEL2.predict([X_val], batch_size=1024, verbose=1)
 thresholds = []
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 res = metrics.f1_score(y_val,(pred_val_y_2 > thresh ).astype(int))
thresholds.append([thresh, res])
 print("F1 score at threshold {0} is {1}".format(thresh, res))
thresholds.sort(key=lambda x: x[1], reverse=True)
 best_thresh_2 = thresholds[0][0]
print("Best threshold: ", best_thresh_2)
 y_pred_2 = MODEL2.predict(x_test, batch_size=1024, verbose=True )<choose_model_class>
missing=(all_data.isnull().sum().sort_values(ascending=False)/len(all_data)) [:35]
missing_data = pd.DataFrame({'Missing Ratio' :missing})
 missing_data.head(5 )
<choose_model_class>
all_data['MSSubClass']=all_data['MSSubClass'].apply(str)
 all_data['OverallCond'] = all_data['OverallCond'].astype(str)
 all_data['YrSold']=all_data['YrSold'].apply(str)
 all_data['MoSold']=all_data['MoSold'].apply(str )
def model4() :
inp = Input(shape=(maxlen,))
embed = Embedding(max_features, embed_size * 3, weights=[embedding_matrix], trainable=False )(inp)
 x = embed
x = Bidirectional(CuDNNGRU(128, return_sequences=True))(x)
 x = attention_3d_block(x)
 x = Bidirectional(CuDNNGRU(128, return_sequences=True))(x)
 x = AttLayer(64 )(x)
 x = Dropout(0.3 )(x)
 x = Dense(128, activation='relu' )(x)
 outp = Dense(1, activation="sigmoid" )(x)
 outp = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=outp)
 model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
 return model<train_model>
cols_for_lab_enc=('FireplaceQu', 'BsmtQual', 'BsmtCond', 'GarageQual', 'GarageCond',
'ExterQual', 'ExterCond','HeatingQC', 'PoolQC', 'KitchenQual', 'BsmtFinType1',
'BsmtFinType2', 'Functional', 'Fence', 'BsmtExposure', 'GarageFinish', 'LandSlope',
'LotShape', 'PavedDrive', 'Street', 'Alley', 'CentralAir', 'MSSubClass', 'OverallCond',
'YrSold', 'MoSold')
 for c in cols_for_lab_enc:
le=LabelEncoder()
le.fit(list(all_data[c].values))
all_data[c] = le.transform(list(all_data[c].values))
MODEL4 = model4()
MODEL4.summary()
batch_size = 1536
epochs = 3
early_stopping = EarlyStopping(patience=3, verbose=1, monitor='val_loss', mode='min')
 model_checkpoint = ModelCheckpoint('./model4.model', save_best_only=True, verbose=1, monitor='val_loss', mode='min')
 reduce_lr = ReduceLROnPlateau(factor=0.5, patience=3, min_lr=0.0001, verbose=1)
 hist = MODEL4.fit(X_tra, y_tra, batch_size=batch_size, epochs=epochs, validation_data=(X_val, y_val), verbose=True)
 MODEL4.save('./model4.h5' )<find_best_params>
all_data['TotalSF'] = all_data['TotalBsmtSF'] + all_data['1stFlrSF'] + all_data['2ndFlrSF']
all_data['haspool'] = all_data['PoolArea'].apply(lambda x: 1 if x > 0 else 0)
 all_data['has2ndfloor'] = all_data['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)
 all_data['hasgarage'] = all_data['GarageArea'].apply(lambda x: 1 if x > 0 else 0)
 all_data['hasbsmt'] = all_data['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 all_data['hasfireplace'] = all_data['Fireplaces'].apply(lambda x: 1 if x > 0 else 0)
 all_data['Total_sqr_footage'] =(all_data['BsmtFinSF1'] + all_data['BsmtFinSF2'] +
all_data['1stFlrSF'] + all_data['2ndFlrSF'])
 all_data['Total_Bathrooms'] =(all_data['FullBath'] +(0.5 * all_data['HalfBath'])+
all_data['BsmtFullBath'] +(0.5 * all_data['BsmtHalfBath']))
all_data['Total_porch_sf'] =(all_data['OpenPorchSF'] + all_data['3SsnPorch'] +
all_data['EnclosedPorch'] + all_data['ScreenPorch'] +
all_data['WoodDeckSF'] )
pred_val_y_4 = MODEL4.predict([X_val], batch_size=1024, verbose=1)
 thresholds = []
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 res = metrics.f1_score(y_val,(pred_val_y_4 > thresh ).astype(int))
thresholds.append([thresh, res])
 print("F1 score at threshold {0} is {1}".format(thresh, res))
thresholds.sort(key=lambda x: x[1], reverse=True)
 best_thresh_4 = thresholds[0][0]
print("Best threshold: ", best_thresh_4)
 y_pred_4 = MODEL4.predict(x_test, batch_size=1024, verbose=True )<define_search_model>
num_feat=all_data.dtypes[all_data.dtypes!="object"].index
num_feat
def model5() :
inp = Input(shape=(maxlen,))
embed = Embedding(max_features, embed_size * 3, weights=[embedding_matrix], trainable=False )(inp)
 x = embed
x0 = Bidirectional(CuDNNGRU(128, return_sequences=True))(x)
 x1 = attention_3d_block(x0)
 x2 = Bidirectional(CuDNNGRU(128, return_sequences=True))(x1)
 x3 = Add()([x0, x2])
 x4 = Bidirectional(CuDNNGRU(64, return_sequences=True))(x3)
 x5 = AttLayer(64 )(x4)
 x = Dropout(0.3 )(x5)
 x = Dense(128, activation='relu' )(x)
 outp = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=outp)
 model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
 return model<train_model>
skews_col=all_data[num_feat].skew().sort_values(ascending=False)
 skews_col = skews_col[abs(skews_col)> 0.75]
skews_col
MODEL5 = model5()
MODEL5.summary()
batch_size = 1536
epochs = 3
early_stopping = EarlyStopping(patience=3, verbose=1, monitor='val_loss', mode='min')
 model_checkpoint = ModelCheckpoint('./model5.model', save_best_only=True, verbose=1, monitor='val_loss', mode='min')
 reduce_lr = ReduceLROnPlateau(factor=0.5, patience=3, min_lr=0.0001, verbose=1)
 hist = MODEL5.fit(X_tra, y_tra, batch_size=batch_size, epochs=epochs, validation_data=(X_val, y_val), verbose=True)
 MODEL5.save('./model5.h5' )<find_best_params>
for c in skews_col.index:
all_data[c]=boxcox1p(all_data[c],0.1)
pred_val_y_5 = MODEL5.predict([X_val], batch_size=1024, verbose=1)
 thresholds = []
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 res = metrics.f1_score(y_val,(pred_val_y_5 > thresh ).astype(int))
thresholds.append([thresh, res])
 print("F1 score at threshold {0} is {1}".format(thresh, res))
thresholds.sort(key=lambda x: x[1], reverse=True)
 best_thresh_5 = thresholds[0][0]
print("Best threshold: ", best_thresh_5)
 y_pred_5 = MODEL5.predict(x_test, batch_size=1024, verbose=True )<find_best_params>
skews_col=all_data[num_feat].skew().sort_values(ascending=False)
 skews_col = skews_col[abs(skews_col)> 0.5]
skews_col
pred_val_y =(3*pred_val_y_2 + 4*pred_val_y_4 + 3*pred_val_y_5)/10
thresholds = []
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 res = metrics.f1_score(y_val,(pred_val_y > thresh ).astype(int))
thresholds.append([thresh, res])
 print("F1 score at threshold {0} is {1}".format(thresh, res))
thresholds.sort(key=lambda x: x[1], reverse=True)
 best_thresh = thresholds[0][0]
print("Best threshold: ", best_thresh )<save_to_csv>
y=train['SalePrice']
all_data=all_data.drop(['SalePrice'],axis=1)
 all_data=pd.get_dummies(all_data)
 train=all_data[:len(train)]
test=all_data[len(train):]
y_pred =(3*y_pred_2 + 4*y_pred_4 + 3*y_pred_5)/10
y_te =(y_pred[:,0] > best_thresh ).astype(np.int)
 submit_df = pd.DataFrame({"qid": test_df["qid"], "prediction": y_te})
 submit_df.to_csv("submission.csv", index=False )<save_to_csv>
from sklearn.linear_model import ElasticNet, Lasso, BayesianRidge, LassoLarsIC,ElasticNet,Lasso,Ridge
from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor
from sklearn.kernel_ridge import KernelRidge
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import RobustScaler
from sklearn.base import BaseEstimator, TransformerMixin, RegressorMixin, clone
from sklearn.model_selection import KFold, cross_val_score, train_test_split,GridSearchCV
from sklearn.metrics import mean_squared_error
from sklearn.neural_network import MLPRegressor
from sklearn.svm import SVR
import xgboost as xgb
import lightgbm as lgb
from bayes_opt import BayesianOptimization
from functools import partial
from sklearn.svm.libsvm import cross_validation
from sklearn.neighbors import KNeighborsRegressor
from mlxtend.regressor import StackingCVRegressor
def create_download_link(df, title = "Download CSV file", filename = "data.csv"):
csv = df.to_csv(index =False)
 b64 = base64.b64encode(csv.encode())
 payload = b64.decode()
html = '<a download="{filename}" href="data:text/csv;base64,{payload}" target="_blank">{title}</a>'
html = html.format(payload=payload,title=title,filename=filename)
 return HTML(html)
 create_download_link(submit_df )<set_options>
X_train, X_test, y_train, y_test = train_test_split(train, y, test_size=0.2)
 kfolds = KFold(n_splits=10, shuffle=True)
 def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))
def rmsle_cv(model):
rmse= np.sqrt(-cross_val_score(model, train.values, y.values, scoring="neg_mean_squared_error", cv = kfolds.get_n_splits(train.values)))
 return rmse
matplotlib.use("Agg")
 def create_feature_map(features):
outfile = open('xgb.fmap', 'w')
 for i, feat in enumerate(features):
outfile.write('{0}\t{1}\tq
'.format(i, feat))
outfile.close()
def rmspe(y, yhat):
return np.sqrt(np.mean(( yhat/y-1)** 2))
def rmspe_xg(yhat, y):
y = np.expm1(y.get_label())
 yhat = np.expm1(yhat)
 return "rmspe", rmspe(y,yhat)
 def build_features(features, data):
data.fillna(0, inplace=True)
 data.loc[data.Open.isnull() , 'Open'] = 1
features.extend(['Store', 'CompetitionDistance', 'CompetitionOpenSinceMonth',
'CompetitionOpenSinceYear', 'Promo', 'Promo2', 'Promo2SinceWeek',
'Promo2SinceYear'])
 features.append('SchoolHoliday')
 data['SchoolHoliday'] = data['SchoolHoliday'].astype(float)
 features.extend(['StoreType', 'Assortment', 'StateHoliday'])
 mappings = {'0':0, 'a':1, 'b':2, 'c':3, 'd':4}
data.StoreType.replace(mappings, inplace=True)
 data.Assortment.replace(mappings, inplace=True)
 data.StateHoliday.replace(mappings, inplace=True)
 features.extend(['DayOfWeek', 'month', 'day', 'year'])
 data['year'] = data.Date.dt.year
data['month'] = data.Date.dt.month
data['day'] = data.Date.dt.day
data['DayOfWeek'] = data.Date.dt.dayofweek 
<load_from_csv>
class StackingAveragedModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, base_models, meta_model, n_folds=5):
self.base_models = base_models
self.meta_model = meta_model
self.n_folds = n_folds
def fit(self, X, y):
self.base_models_ = [list() for x in self.base_models]
self.meta_model_ = clone(self.meta_model)
 kfold = KFold(n_splits=self.n_folds, shuffle=True, random_state=156)
 out_of_fold_predictions = np.zeros(( X.shape[0], len(self.base_models)))
 for i, model in enumerate(self.base_models):
for train_index, holdout_index in kfold.split(X, y):
instance = clone(model)
 self.base_models_[i].append(instance)
 instance.fit(X[train_index], y[train_index])
 y_pred = instance.predict(X[holdout_index])
 out_of_fold_predictions[holdout_index, i] = y_pred
self.meta_model_.fit(out_of_fold_predictions, y)
 return self
def predict(self, X):
meta_features = np.column_stack([
np.column_stack([model.predict(X)for model in base_models] ).mean(axis=1)
 for base_models in self.base_models_ ])
 return self.meta_model_.predict(meta_features )
print("Load the training, test and store data using pandas")
 types = {'CompetitionOpenSinceYear': np.dtype(int),
'CompetitionOpenSinceMonth': np.dtype(int),
'StateHoliday': np.dtype(str),
'Promo2SinceWeek': np.dtype(int),
'SchoolHoliday': np.dtype(float),
'PromoInterval': np.dtype(str)}
train = pd.read_csv(".. /input/train.csv", parse_dates=[2], dtype=types)
 test = pd.read_csv(".. /input/test.csv", parse_dates=[3], dtype=types)
 store = pd.read_csv(".. /input/store.csv")
 print("Assume store open, if not provided")
 train.fillna(1, inplace=True)
 test.fillna(1, inplace=True)
 print("Consider only open stores for training.Closed stores wont count into the score.")
 train = train[train["Open"] != 0]
print("Use only Sales bigger then zero.Simplifies calculation of rmspe")
 train = train[train["Sales"] > 0]
print("Join with store")
 train = pd.merge(train, store, on='Store')
 test = pd.merge(test, store, on='Store')
<categorify>
class AvarageModels(BaseEstimator, TransformerMixin, RegressorMixin):
def __init__(self,models):
self.models=models
def fit(self,X,y):
self.models_ = [clone(x)for x in self.models]
for model in self.models_:
model.fit(X, y)
 return self
def predict(self,X):
prediction= np.column_stack([m.predict(X)for m in self.models_])
 return np.mean(prediction,axis=1 )
features = []
print("augment features")
 build_features(features, train)
 build_features([], test)
 print(features)
 print('training data processed')
<train_model>
class StackNet(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, base_models,meta_final_model, meta_models1=None, meta_models2=None,add_prev_out=True, n_folds=10):
self.base_models = base_models
self.meta_models1 = meta_models1
self.meta_models2 = meta_models2
self.meta_final_model=meta_final_model
self.n_folds = n_folds
self.add_prev_out=add_prev_out
def fit(self, X, y):
self.base_models_ = [list() for x in self.base_models]
if self.meta_models1!=None:
self.meta_models1_ = [list() for x in self.meta_models1]
if self.meta_models2!=None:
self.meta_models2_ = [list() for x in self.meta_models2]
self.meta_final_model_=clone(self.meta_final_model)
 kfold = KFold(n_splits=self.n_folds, shuffle=True, random_state=156)
 out_of_fold_predictions = np.zeros(( X.shape[0], len(self.base_models)))
 for i, model in enumerate(self.base_models):
for train_index, holdout_index in kfold.split(X, y):
instance = clone(model)
 self.base_models_[i].append(instance)
 instance.fit(X[train_index], y[train_index])
 y_pred = instance.predict(X[holdout_index])
 out_of_fold_predictions[holdout_index, i] = y_pred
if self.add_prev_out:
out_of_fold_predictions=np.hstack(( out_of_fold_predictions,X))
out_of_fold_predictions2 = np.zeros(( X.shape[0],len(self.meta_models1)))
 if self.meta_models1!=None:
for i, model in enumerate(self.meta_models1):
for train_index, holdout_index in kfold.split(out_of_fold_predictions, y):
instance = clone(model)
 self.meta_models1_[i].append(instance)
 instance.fit(out_of_fold_predictions[train_index], y[train_index])
 y_pred = instance.predict(out_of_fold_predictions[holdout_index])
 out_of_fold_predictions2[holdout_index, i] = y_pred
if self.add_prev_out:
out_of_fold_predictions2=np.hstack(( out_of_fold_predictions2,X))
else:
out_of_fold_predictions2=out_of_fold_predictions
out_of_fold_predictions3 = np.zeros(( X.shape[0],len(self.meta_models2)))
 if self.meta_models2!=None:
for i, model in enumerate(self.meta_models2):
for train_index, holdout_index in kfold.split(out_of_fold_predictions2, y):
instance = clone(model)
 self.meta_models2_[i].append(instance)
 instance.fit(out_of_fold_predictions2[train_index], y[train_index])
 y_pred = instance.predict(out_of_fold_predictions2[holdout_index])
 out_of_fold_predictions3[holdout_index, i] = y_pred
if self.add_prev_out:
out_of_fold_predictions3=np.hstack(( out_of_fold_predictions3,X))
else:
out_of_fold_predictions3=out_of_fold_predictions2
self.meta_final_model_.fit(out_of_fold_predictions3, y)
 return self
def predict(self, X):
meta_features = np.column_stack([
np.column_stack([model.predict(X)for model in base_models] ).mean(axis=1)
 for base_models in self.base_models_])
 if self.add_prev_out:
meta_features=np.hstack(( meta_features,X))
if self.meta_models1!=None:
meta_features2 = np.column_stack([
np.column_stack([model.predict(meta_features)for model in meta_models1_] ).mean(axis=1)
 for meta_models1_ in self.meta_models1_ ])
 if self.add_prev_out:
meta_features2=np.hstack(( meta_features2,X))
else:
meta_features2=meta_features
if self.meta_models2!=None:
meta_features3 = np.column_stack([
np.column_stack([model.predict(meta_features2)for model in meta_models2_] ).mean(axis=1)
 for meta_models2_ in self.meta_models2_ ])
 if self.add_prev_out:
meta_features3=np.hstack(( meta_features3,X))
else:
meta_features3=meta_features2
return self.meta_final_model_.predict(meta_features3 )
params = {"objective": "reg:linear",
"booster" : "gbtree",
"eta": 0.09,
"max_depth": 10,
"subsample": 0.9,
"colsample_bytree": 0.7,
"silent": 1,
"seed": 1301
}
num_boost_round = 800
print("Train a XGBoost model")
 X_train, X_valid = train_test_split(train, test_size=0.012, random_state=10)
 y_train = np.log1p(X_train.Sales)
 y_valid = np.log1p(X_valid.Sales)
 dtrain = xgb.DMatrix(X_train[features], y_train)
 dvalid = xgb.DMatrix(X_valid[features], y_valid)
 watchlist = [(dtrain, 'train'),(dvalid, 'eval')]
gbm = xgb.train(params, dtrain, num_boost_round, evals=watchlist, \
early_stopping_rounds=100, feval=rmspe_xg, verbose_eval=True)
 print("Validating")
 yhat = gbm.predict(xgb.DMatrix(X_valid[features]))
error = rmspe(X_valid.Sales.values, np.expm1(yhat))
print('RMSPE: {:.6f}'.format(error))
print("Make predictions on the test set")
 dtest = xgb.DMatrix(test[features])
 test_probs = gbm.predict(dtest)
 result = pd.DataFrame({"Id": test["Id"], 'Sales': np.expm1(test_probs)})
 result.to_csv("xgboost_10_submission.csv", index=False)
 create_feature_map(features)
 importance = gbm.get_fscore(fmap='xgb.fmap')
 importance = sorted(importance.items() , key=operator.itemgetter(1))
df = pd.DataFrame(importance, columns=['feature', 'fscore'])
 df['fscore'] = df['fscore'] / df['fscore'].sum()
featp = df.plot(kind='barh', x='feature', y='fscore', legend=False, figsize=(6, 10))
plt.title('XGBoost Feature Importance')
 plt.xlabel('relative importance')
 fig_featp = featp.get_figure()
fig_featp.savefig('feature_importance_xgb.png', bbox_inches='tight', pad_inches=1 )<import_modules>
max_iter=1e7
l1_ratio=[x/100 for x in range(65,100)]
alphas=[y*(10**-x)for x in range(2,6)for y in range(1,9)]
C=[x for x in range(1,30,3)]
epsilon=[y*(10**-x)for x in range(2,6)for y in range(1,9)]
gamma=[y*(10**-x)for x in range(2,6)for y in range(1,9)]
xgb_1=xgb.XGBRegressor(colsample_bytree=0.4603, gamma=0.0468,
learning_rate=0.05, max_depth=3,
min_child_weight=1.7817, n_estimators=2200,
reg_alpha=0.4640, reg_lambda=0.8571,
subsample=0.5213, silent=1,
random_state =7, nthread = -1)
 lgb_1=lgb.LGBMRegressor(objective='regression',num_leaves=5,
learning_rate=0.05, n_estimators=720,
max_bin = 55, bagging_fraction = 0.8,
bagging_freq = 5, feature_fraction = 0.2319,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf =6, min_sum_hessian_in_leaf = 11)
 enet=make_pipeline(RobustScaler() ,ElasticNet(alpha=0.0005, l1_ratio=.9))
lasso=make_pipeline(RobustScaler() ,Lasso(alpha =0.0005))
GBoost = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05,
max_depth=4, max_features='sqrt',
min_samples_leaf=15, min_samples_split=10,
loss='huber', random_state =5)
 KRR = KernelRidge(alpha=0.6, kernel='polynomial', degree=2, coef0=2.5 )
pt_models = ".. /input/pretrained-models/pretrained-models.pytorch-master/"
sys.path.insert(0,pt_models)
<define_search_space>
w=rmsle_cv(enet)
 print(w.mean() ,w.std())
 w=rmsle_cv(lasso)
 print(w.mean() ,w.std())
 w=rmsle_cv(GBoost)
 print(w.mean() ,w.std())
 w=rmsle_cv(KRR)
 print(w.mean() ,w.std())
 w=rmsle_cv(am)
 print(w.mean() ,w.std())
 w=rmsle_cv(sav)
 print(w.mean() ,w.std())
 xgb_1.fit(X_train.values,y_train.values)
 w=rmsle(xgb_1.predict(X_test.values),y_test.values)
 print(w)
 lgb_1.fit(X_train.values,y_train.values)
 w=rmsle(lgb_1.predict(X_test.values),y_test.values)
 print(w)
 sn.fit(X_train.values,y_train.values)
 w=rmsle(sn.predict(X_test.values),y_test.values)
 print(w )
TEST_BATCH_SIZE = 32
MODEL_MEAN=(0.485,0.465,0.406)
 MODEL_STD=(0.229,0.224,0.225)
 IMG_HEIGHT=137
IMG_WIDTH=236
DEVICE="cuda"<choose_model_class>
xgb_1.fit(train,y)
 lgb_1.fit(train,y)
 sn.fit(train.values,y.values)
 predicted_prices=np.expm1(sn.predict(test.values)) *0.7+np.expm1(xgb_1.predict(test)) *0.00+np.expm1(lgb_1.predict(test)) *0.3
my_submission = pd.DataFrame({'Id': test_Id, 'SalePrice': predicted_prices})
 my_submission.to_csv('submission.csv', index=False)