Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<choose_model_class>
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
from scipy import stats
from scipy.stats import norm, skew
class BiLSTM(nn.Module):
def __init__(self, pretrained_lm, padding_idx, static=True, hidden_dim=128, lstm_layer=2, dropout=0.2):
super(BiLSTM, self ).__init__()
self.hidden_dim = hidden_dim
self.dropout = nn.Dropout(p=dropout)
 self.embedding = nn.Embedding.from_pretrained(pretrained_lm)
 self.embedding.padding_idx = padding_idx
if static:
self.embedding.weight.requires_grad = False
self.lstm = nn.LSTM(input_size=self.embedding.embedding_dim,
hidden_size=hidden_dim,
num_layers=lstm_layer,
dropout = dropout,
bidirectional=True)
 self.hidden2label = nn.Linear(hidden_dim*lstm_layer*2, 1)
 def forward(self, sents):
x = self.embedding(sents)
 x = torch.transpose(x, dim0=1, dim1=0)
 lstm_out,(h_n, c_n)= self.lstm(x)
 y = self.hidden2label(self.dropout(torch.cat([c_n[i,:, :] for i in range(c_n.shape[0])], dim=1)))
 return y<train_model>
test = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/test.csv")
 train = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/train.csv")
def training(epoch, model, eval_every, loss_func, optimizer, train_iter, val_iter, early_stop=1, warmup_epoch=2):
step = 0
max_loss = 1e5
no_improve_epoch = 0
no_improve_in_previous_epoch = False
fine_tuning = False
train_record = []
val_record = []
losses = []
for e in range(epoch):
if e >= warmup_epoch:
if no_improve_in_previous_epoch:
no_improve_epoch += 1
if no_improve_epoch >= early_stop:
break
else:
no_improve_epoch = 0
no_improve_in_previous_epoch = True
if not fine_tuning and e >= warmup_epoch:
model.embedding.weight.requires_grad = True
fine_tuning = True
train_iter.init_epoch()
for train_batch in iter(train_iter):
step += 1
model.train()
x = train_batch.text.cuda()
y = train_batch.target.type(torch.Tensor ).cuda()
model.zero_grad()
pred = model.forward(x ).view(-1)
 loss = loss_function(pred, y)
 losses.append(loss.cpu().data.numpy())
 train_record.append(loss.cpu().data.numpy())
 loss.backward()
optimizer.step()
if step % eval_every == 0:
model.eval()
model.zero_grad()
val_loss = []
for val_batch in iter(val_iter):
val_x = val_batch.text.cuda()
val_y = val_batch.target.type(torch.Tensor ).cuda()
val_pred = model.forward(val_x ).view(-1)
 val_loss.append(loss_function(val_pred, val_y ).cpu().data.numpy())
 val_record.append({'step': step, 'loss': np.mean(val_loss)})
 print('epcoh {:02} - step {:06} - train_loss {:.4f} - val_loss {:.4f} '.format(
 e, step, np.mean(losses), val_record[-1]['loss']))
if e >= warmup_epoch:
if val_record[-1]['loss'] <= max_loss:
save(m=model, info={'step': step, 'epoch': e, 'train_loss': np.mean(losses),
'val_loss': val_record[-1]['loss']})
 max_loss = val_record[-1]['loss']
no_improve_in_previous_epoch = False
def save(m, info):
torch.save(info, 'best_model.info')
 torch.save(m, 'best_model.m')
 def load() :
m = torch.load('best_model.m')
 info = torch.load('best_model.info')
 return m, info 
<define_variables>
data = train
data = data.select_dtypes(exclude=['object'])
 data = data.dropna(axis=0)
 y = data["SalePrice"]
X = data.drop(["SalePrice"], axis=1 )
batch_size = 128
train_iter = torchtext.data.BucketIterator(dataset=train,
batch_size=batch_size,
sort_key=lambda x: x.text.__len__() ,
shuffle=True,
sort=False)
 val_iter = torchtext.data.BucketIterator(dataset=val,
batch_size=batch_size,
sort_key=lambda x: x.text.__len__() ,
train=False,
sort=False)
 model = BiLSTM(text.vocab.vectors, lstm_layer=2, padding_idx=text.vocab.stoi[text.pad_token], hidden_dim=128 ).cuda()
loss_function = nn.BCEWithLogitsLoss()
optimizer = optim.Adam(filter(lambda p: p.requires_grad, model.parameters()),
lr=1e-3 )<train_model>
from sklearn.model_selection import KFold, cross_val_score, train_test_split
from sklearn.ensemble import RandomForestRegressor
from xgboost import XGBRegressor
from sklearn.metrics import mean_absolute_error
training(model=model, epoch=20, eval_every=500,
loss_func=loss_function, optimizer=optimizer, train_iter=train_iter,
val_iter=val_iter, warmup_epoch=3, early_stop=2 )<load_pretrained>
data.isnull().sum().sum()
model, m_info = load()
m_info<find_best_params>
X_train, X_test, y_train, y_test = train_test_split(X, y )
model.lstm.flatten_parameters()<predict_on_test>
model_rf1 = RandomForestRegressor(n_estimators = 500, random_state=17)
 model_rf1.fit(X_train, y_train )
model.eval()
val_pred = []
val_true = []
val_iter.init_epoch()
for val_batch in iter(val_iter):
val_x = val_batch.text.cuda()
val_true += val_batch.target.data.numpy().tolist()
val_pred += torch.sigmoid(model.forward(val_x ).view(-1)).cpu().data.numpy().tolist()<compute_test_metric>
rf1_pred = model_rf1.predict(X_test )
tmp = [0,0,0]
delta = 0
for tmp[0] in np.arange(0.1, 0.501, 0.01):
tmp[1] = f1_score(val_true, np.array(val_pred)>tmp[0])
 if tmp[1] > tmp[2]:
delta = tmp[0]
tmp[2] = tmp[1]
print('best threshold is {:.4f} with F1 score: {:.4f}'.format(delta, tmp[2]))<find_best_params>
mae_rf1 = mean_absolute_error(rf1_pred, y_test)
 print(mae_rf1 )
model.eval()
model.zero_grad()
test_iter = torchtext.data.BucketIterator(dataset=test,
batch_size=batch_size,
sort_key=lambda x: x.text.__len__() ,
sort=True)
 test_pred = []
test_id = []
for test_batch in iter(test_iter):
test_x = test_batch.text.cuda()
test_pred += torch.sigmoid(model.forward(test_x ).view(-1)).cpu().data.numpy().tolist()
test_id += test_batch.qid.view(-1 ).data.numpy().tolist() 
<create_dataframe>
model_xgb1 = XGBRegressor(n_estimators=500, learning_rate=0.1 ,random_state = 17)
 model_xgb1.fit(X_train, y_train )
sub_df =pd.DataFrame()
sub_df['qid'] = [qid.vocab.itos[i] for i in test_id]
sub_df['prediction'] =(np.array(test_pred)>= delta ).astype(int )<save_to_csv>
xgb1_pred = model_xgb1.predict(X_test )
sub_df.to_csv("submission.csv", index=False )<define_variables>
mae_xgb1 = mean_absolute_error(xgb1_pred, y_test)
 print(mae_xgb1 )
embed_size = 300
max_features = 120000
maxlen = 70<import_modules>
predictor_cols = X_train.columns
test = test[predictor_cols]
xgb1_finalpred = model_xgb1.predict(test)
 print(xgb1_finalpred)
 my_submission = pd.DataFrame({'Id': test.Id, 'SalePrice': xgb1_finalpred})
 my_submission.to_csv('xgb1_pred.csv', index=False )
import os
import time
import numpy as np
import pandas as pd
from tqdm import tqdm
import math
from sklearn.model_selection import train_test_split
from sklearn import metrics
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.layers import Dense, Input, CuDNNLSTM, Embedding, Dropout, Activation, CuDNNGRU, Conv1D
from keras.layers import Bidirectional, GlobalMaxPool1D, GlobalMaxPooling1D, GlobalAveragePooling1D
from keras.layers import Input, Embedding, Dense, Conv2D, MaxPool2D, concatenate
from keras.layers import Reshape, Flatten, Concatenate, Dropout, SpatialDropout1D
from keras.optimizers import Adam
from keras.models import Model
from keras import backend as K
from keras.engine.topology import Layer
from keras import initializers, regularizers, constraints, optimizers, layers<load_from_csv>
test = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/test.csv")
 train = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/train.csv")
 train_ID = train['Id']
test_ID = test['Id']
train.drop("Id", axis = 1, inplace = True)
 test.drop("Id", axis = 1, inplace = True)
 ntrain = train.shape[0]
ntest = test.shape[0]
y = train["SalePrice"]
data = pd.concat([train, test], sort = True ).reset_index(drop=True )
def load_and_prec() :
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 train_df["question_text"] = train_df["question_text"].str.lower()
test_df["question_text"] = test_df["question_text"].str.lower()
train_df["question_text"] = train_df["question_text"].apply(lambda x: clean_text(x))
test_df["question_text"] = test_df["question_text"].apply(lambda x: clean_text(x))
print("Train shape : ",train_df.shape)
 print("Test shape : ",test_df.shape)
 train_df, val_df = train_test_split(train_df, test_size=0.001, random_state=2018)
 train_X = train_df["question_text"].fillna("_
val_X = val_df["question_text"].fillna("_
test_X = test_df["question_text"].fillna("_
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 val_X = tokenizer.texts_to_sequences(val_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 train_X = pad_sequences(train_X, maxlen=maxlen)
 val_X = pad_sequences(val_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 train_y = train_df['target'].values
val_y = val_df['target'].values
np.random.seed(2018)
 trn_idx = np.random.permutation(len(train_X))
val_idx = np.random.permutation(len(val_X))
train_X = train_X[trn_idx]
val_X = val_X[val_idx]
train_y = train_y[trn_idx]
val_y = val_y[val_idx]
return train_X, val_X, test_X, train_y, val_y, tokenizer.word_index<statistical_test>
data.drop(['SalePrice'], axis=1, inplace=True )
def load_glove(word_index):
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = -0.005838499,0.48782197
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_fasttext(word_index):
EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_para(word_index):
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = -0.0053247833,0.49346462
embed_size = all_embs.shape[1]
print(emb_mean,emb_std,"para")
 nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix<choose_model_class>
nullcols = data.isnull().sum().sort_values(ascending=False)
 nullcols = pd.DataFrame(nullcols, columns=["Total"])
 nullcols[nullcols["Total"] > 0]
def model_cnn(embedding_matrix):
filter_sizes = [1,2,3,5]
num_filters = 36
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Reshape(( maxlen, embed_size, 1))(x)
 maxpool_pool = []
for i in range(len(filter_sizes)) :
conv = Conv2D(num_filters, kernel_size=(filter_sizes[i], embed_size),
kernel_initializer='he_normal', activation='elu' )(x)
 maxpool_pool.append(MaxPool2D(pool_size=(maxlen - filter_sizes[i] + 1, 1))(conv))
z = Concatenate(axis=1 )(maxpool_pool)
 z = Flatten()(z)
 z = Dropout(0.1 )(z)
 outp = Dense(1, activation="sigmoid" )(z)
 model = Model(inputs=inp, outputs=outp)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model<choose_model_class>
data.drop("PoolQC", axis = 1, inplace = True)
 data.drop("MiscFeature", axis = 1, inplace = True)
 data.drop("Alley", axis = 1, inplace = True)
 data.drop("Fence", axis = 1, inplace = True)
 data = data.drop(['Utilities'], axis=1)
 data['FireplaceQu'] = data['FireplaceQu'].fillna("No")
 data['LotFrontage'] = data['LotFrontage'].fillna(0)
 data['GarageCond'] = data['GarageCond'].fillna("No")
 data['GarageType'] = data['GarageType'].fillna("No")
 data['GarageYrBlt'].fillna(( data['GarageYrBlt'].mean()), inplace=True)
 data['GarageFinish'] = data['GarageFinish'].fillna("No")
 data['GarageQual'] = data['GarageQual'].fillna("No")
 data['BsmtExposure'] = data['BsmtExposure'].fillna("No")
 data['BsmtFinType2'] = data['BsmtFinType2'].fillna("No")
 data['BsmtFinType1'] = data['BsmtFinType1'].fillna("No")
 data['BsmtCond'] = data['BsmtCond'].fillna("No")
 data['BsmtQual'] = data['BsmtQual'].fillna("No")
 data['MasVnrArea'] = data['MasVnrArea'].fillna("No")
 data['MasVnrType'] = data['MasVnrType'].fillna("No")
 data['Electrical'] = data['Electrical'].fillna("SBrkr")
 data['MSZoning'] = data['MSZoning'].fillna(data['MSZoning'].mode() [0])
 for col in('BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF','TotalBsmtSF', 'BsmtFullBath', 'BsmtHalfBath'):
data[col] = data[col].fillna(0)
 data["Functional"] = data["Functional"].fillna("Typ")
 data['Exterior1st'] = data['Exterior1st'].fillna(data['Exterior1st'].mode() [0])
 data['Exterior2nd'] = data['Exterior2nd'].fillna(data['Exterior2nd'].mode() [0])
 for col in('GarageArea', 'GarageCars'):
data[col] = data[col].fillna(0)
 data['KitchenQual'] = data['KitchenQual'].fillna(data['KitchenQual'].mode() [0])
 data['SaleType'] = data['SaleType'].fillna(data['SaleType'].mode() [0] )
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim<choose_model_class>
data.isnull().sum().sum()
def model_lstm_atten(embedding_matrix):
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)
 x = Attention(maxlen )(x)
 x = Dense(64, activation="relu" )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model<choose_model_class>
data['MSSubClass'] = data['MSSubClass'].apply(str)
 data['OverallCond'] = data['OverallCond'].astype(str)
 data['YrSold'] = data['YrSold'].astype(str)
 data['MoSold'] = data['MoSold'].astype(str )
def model_gru_ap_atten(embedding_matrix):
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)
 x = Attention(maxlen )(x)
 x = Dense(16, activation="relu" )(x)
 x = Dropout(0.1 )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model 
<choose_model_class>
cols =('BsmtQual', 'BsmtCond', 'GarageQual', 'GarageCond',
'ExterQual', 'ExterCond','HeatingQC', 'KitchenQual', 'BsmtFinType1',
'BsmtFinType2', 'Functional', 'BsmtExposure', 'GarageFinish', 'LandSlope',
'LotShape', 'PavedDrive', 'Street', 'CentralAir', 'MSSubClass', 'OverallCond',
'YrSold', 'MoSold')
 for col in cols:
encoder = LabelEncoder()
encoder.fit(list(data[col].values))
data[col] = encoder.transform(list(data[col].values))
data['TotalSF'] = data['TotalBsmtSF'] + data['1stFlrSF'] + data['2ndFlrSF']
data.info()
def model_lstm_du(embedding_matrix):
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)
 avg_pool = GlobalAveragePooling1D()(x)
 max_pool = GlobalMaxPooling1D()(x)
 conc = concatenate([avg_pool, max_pool])
 conc = Dense(64, activation="relu" )(conc)
 conc = Dropout(0.1 )(conc)
 outp = Dense(1, activation="sigmoid" )(conc)
 model = Model(inputs=inp, outputs=outp)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model<choose_model_class>
skewness = skewness[abs(skewness)>0.75]
print("There are {} skewed numerical features to Box Cox transform".format(skewness.shape[0]))
skewed_features = skewness.index
lam = 0.15
for feat in skewed_features:
data[feat] = boxcox1p(data[feat], lam )
def model_gru_atten_3(embedding_matrix):
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = Bidirectional(CuDNNGRU(128, return_sequences=True))(x)
 x = Bidirectional(CuDNNGRU(100, return_sequences=True))(x)
 x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)
 x = Attention(maxlen )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model<load_pretrained>
data = pd.get_dummies(data)
 data.info()
train_X, val_X, test_X, train_y, val_y, word_index = load_and_prec()
vocab = []
for w,k in word_index.items() :
vocab.append(w)
 if k >= max_features:
break
embedding_matrix_1 = load_glove(word_index)
 embedding_matrix_3 = load_para(word_index )<define_variables>
train = data[:ntrain]
test = data[ntrain:]
X = train
X_train, X_test, y_train, y_test = train_test_split(X, y )
outputs = []<predict_on_test>
model_xgb2 = XGBRegressor(n_estimators=500, learning_rate=0.1 ,random_state = 17 )
pred_val_y, pred_test_y = train_pred(model_gru_atten_3(embedding_matrix), epochs = 3)
 outputs.append([pred_val_y, pred_test_y, '3 GRU w/ atten'] )<predict_on_test>
model_xgb2.fit(X_train, y_train )
pred_val_y, pred_test_y = train_pred(model_gru_ap_atten(embedding_matrix), epochs = 3)
 outputs.append([pred_val_y, pred_test_y, 'gru atten ap'] )<predict_on_test>
xgb2_pred = model_xgb2.predict(X_test)
 mae_xgb2 = mean_absolute_error(xgb2_pred, y_test)
 print(mae_xgb2 )
pred_val_y, pred_test_y = train_pred(model_cnn(embedding_matrix_1), epochs = 3)
 outputs.append([pred_val_y, pred_test_y, '2d CNN GloVe'] )<predict_on_test>
predictor_cols = X_train.columns
test = test[predictor_cols]
xgb2_finalpred = model_xgb2.predict(test)
 print(xgb2_finalpred)
 my_submission2 = pd.DataFrame({'Id': test_ID, 'SalePrice': xgb2_finalpred})
 my_submission2.to_csv('xgb2_pred.csv', index=False )
pred_val_y, pred_test_y = train_pred(model_lstm_du(embedding_matrix), epochs = 3)
 outputs.append([pred_val_y, pred_test_y, 'LSTM DU'] )<predict_on_test>
my_pipeline = Pipeline(steps=[('model', XGBRegressor(colsample_bytree = 0.46, gamma = 0.047,
learning_rate = 0.05, max_depth = 3,
min_child_weight= 1.8, n_estimators = 2200,
reg_alpha = 0.47, reg_lambda = 0.86,
subsample = 0.52, random_state =17))
])
 scores = -1 * cross_val_score(my_pipeline, X_train, y_train,
cv=5,
scoring='neg_mean_absolute_error')
 print("Average MAE score(across experiments):")
 print(scores.mean() )
pred_val_y, pred_test_y = train_pred(model_lstm_atten(embedding_matrix), epochs = 3)
 outputs.append([pred_val_y, pred_test_y, '2 LSTM w/ attention'] )<predict_on_test>
model_xgb3 = XGBRegressor(colsample_bytree = 0.46, gamma = 0.047,
learning_rate = 0.05, max_depth = 3,
min_child_weight= 1.8, n_estimators = 2200,
reg_alpha = 0.47, reg_lambda = 0.86,
subsample = 0.52, random_state =17)
 model_xgb3.fit(X_train, y_train )
pred_val_y, pred_test_y = train_pred(model_lstm_atten(embedding_matrix_1), epochs = 3)
 outputs.append([pred_val_y, pred_test_y, '2 LSTM w/ attention GloVe'] )<predict_on_test>
predictor_cols = X_train.columns
test = test[predictor_cols]
xgb3_finalpred = model_xgb3.predict(test)
 print(xgb3_finalpred)
 my_submission3 = pd.DataFrame({'Id': test_ID, 'SalePrice': xgb3_finalpred})
 my_submission3.to_csv('xgb3_pred.csv', index=False )