Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<train_model>
%matplotlib inline
def ignore_warn(*args, **kwargs):
pass
warnings.warn = ignore_warn
def accuracy(y, t):
pred_label = torch.argmax(y, dim=1)
 count = pred_label.shape[0]
correct =(pred_label == t ).sum().type(torch.float32)
 acc = correct / count
return acc
class BengaliClassifier(nn.Module):
def __init__(self, predictor, n_grapheme=168, n_vowel=11, n_consonant=7):
super(BengaliClassifier, self ).__init__()
self.n_grapheme = n_grapheme
self.n_vowel = n_vowel
self.n_consonant = n_consonant
self.n_total_class = self.n_grapheme + self.n_vowel + self.n_consonant
self.predictor = predictor
self.metrics_keys = [
'loss', 'loss_grapheme', 'loss_vowel', 'loss_consonant',
'acc_grapheme', 'acc_vowel', 'acc_consonant']
def forward(self, x, y=None):
pred = self.predictor(x)
 if isinstance(pred, tuple):
assert len(pred)== 3
preds = pred
else:
assert pred.shape[1] == self.n_total_class
preds = torch.split(pred, [self.n_grapheme, self.n_vowel, self.n_consonant], dim=1)
 loss_grapheme = F.cross_entropy(preds[0], y[:, 0])
 loss_vowel = F.cross_entropy(preds[1], y[:, 1])
 loss_consonant = F.cross_entropy(preds[2], y[:, 2])
 loss = loss_grapheme + loss_vowel + loss_consonant
metrics = {
'loss': loss.item() ,
'loss_grapheme': loss_grapheme.item() ,
'loss_vowel': loss_vowel.item() ,
'loss_consonant': loss_consonant.item() ,
'acc_grapheme': accuracy(preds[0], y[:, 0]),
'acc_vowel': accuracy(preds[1], y[:, 1]),
'acc_consonant': accuracy(preds[2], y[:, 2]),
}
return loss, metrics, pred
def calc(self, data_loader):
device: torch.device = next(self.parameters() ).device
self.eval()
output_list = []
with torch.no_grad() :
for batch in tqdm(data_loader):
batch = batch.to(device)
 pred = self.predictor(batch)
 output_list.append(pred)
 output = torch.cat(output_list, dim=0)
 preds = torch.split(output, [self.n_grapheme, self.n_vowel, self.n_consonant], dim=1)
 return preds
def predict_proba(self, data_loader):
preds = self.calc(data_loader)
 return [F.softmax(p, dim=1)for p in preds]
def predict(self, data_loader):
preds = self.calc(data_loader)
 pred_labels = [torch.argmax(p, dim=1)for p in preds]
return pred_labels 
<choose_model_class>
train = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv')
 test = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv' )
def build_predictor(arch, out_dim, model_name=None):
if arch == 'pretrained':
predictor = PretrainedCNN(in_channels=1, out_dim=out_dim, model_name=model_name)
 else:
raise ValueError("[ERROR] Unexpected value arch={}".format(arch))
return predictor
def build_classifier(arch, load_model_path, n_total, model_name='', device='cuda:0'):
if isinstance(device, str):
device = torch.device(device)
 predictor = build_predictor(arch, out_dim=n_total, model_name=model_name)
 print('predictor', type(predictor))
classifier = BengaliClassifier(predictor)
 if load_model_path:
predictor.load_state_dict(torch.load(load_model_path))
else:
print("[WARNING] Unexpected value load_model_path={}" 
.format(load_model_path))
classifier.to(device)
 return classifier 
<load_pretrained>
train["SalePrice"] = np.log1p(train["SalePrice"])
def prepare_image(datadir, featherdir, data_type='train',
submission=False, indices=[0, 1, 2, 3]):
assert data_type in ['train', 'test']
if submission:
image_df_list = [pd.read_parquet(datadir / f'{data_type}_image_data_{i}.parquet')
 for i in indices]
else:
image_df_list = [pd.read_feather(featherdir / f'{data_type}_image_data_{i}.feather')
 for i in indices]
print('image_df_list', len(image_df_list))
HEIGHT = 137
WIDTH = 236
images = [df.iloc[:, 1:].values.reshape(-1, HEIGHT, WIDTH)for df in image_df_list]
del image_df_list
gc.collect()
images = np.concatenate(images, axis=0)
 return images 
<predict_on_test>
data = pd.concat([train, test] )
def predict_core(test_images, image_size, threshold,
arch, n_total, model_name, load_model_path, batch_size=512, device='cuda:0', **kwargs):
classifier = build_classifier(arch, load_model_path, n_total, model_name, device=device)
 test_dataset = BengaliAIDataset(
 test_images, None,
transform=Transform(affine=False, crop=True, size=(image_size, image_size),
threshold=threshold, train=False))
print('test_dataset', len(test_dataset))
test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)
 test_pred_proba = classifier.predict_proba(test_loader)
 return test_pred_proba 
<define_variables>
data.set_index('Id', inplace = True )
device = torch.device(device)
 n_grapheme = 168
n_vowel = 11
n_consonant = 7
n_total = n_grapheme + n_vowel + n_consonant
print('n_total', n_total )<define_variables>
data.isnull().sum() [:50]
traindir = '/kaggle/input/bengaliaicv19-trainedmodels/'
data_type = 'test'
test_preds_list = []
for i in range(4):
indices = [i]
test_images = prepare_image(
 datadir, featherdir, data_type=data_type, submission=submission, indices=indices)
 n_dataset = len(test_images)
 print(f'n_dataset={n_dataset}')
 model_preds_list = []
for j in range(4):
train_args_dict = load_json(os.path.join(traindir, f'args_{j}.json'))
train_args_dict.update({
'load_model_path': os.path.join(traindir, f'predictor_{j}.pt'),
'device': device,
'batch_size': batch_size,
'debug': debug,
})
 print(f'j {j} updated train_args_dict {train_args_dict}')
 test_preds = predict_core(
 test_images=test_images, n_total=n_total,
**train_args_dict)
 model_preds_list.append(test_preds)
 proba0 = torch.mean(torch.stack([test_preds[0] for test_preds in model_preds_list], dim=0), dim=0)
 proba1 = torch.mean(torch.stack([test_preds[1] for test_preds in model_preds_list], dim=0), dim=0)
 proba2 = torch.mean(torch.stack([test_preds[2] for test_preds in model_preds_list], dim=0), dim=0)
 p0 = torch.argmax(proba0, dim=1 ).cpu().numpy()
p1 = torch.argmax(proba1, dim=1 ).cpu().numpy()
p2 = torch.argmax(proba2, dim=1 ).cpu().numpy()
print('p0', p0.shape, 'p1', p1.shape, 'p2', p2.shape)
 test_preds_list.append([p0, p1, p2])
 if debug:
break
del test_images
gc.collect()<concatenate>
data['PoolQC'].fillna('None', inplace=True)
 data['MiscFeature'].fillna('None', inplace=True)
 data['Alley'].fillna('None', inplace=True)
 data['Fence'].fillna('None', inplace=True)
 data['FireplaceQu'].fillna('None', inplace=True )
p0 = np.concatenate([test_preds[0] for test_preds in test_preds_list], axis=0)
 p1 = np.concatenate([test_preds[1] for test_preds in test_preds_list], axis=0)
 p2 = np.concatenate([test_preds[2] for test_preds in test_preds_list], axis=0)
 print('concat:', 'p0', p0.shape, 'p1', p1.shape, 'p2', p2.shape)
 row_id = []
target = []
for i in tqdm(range(len(p0))):
row_id += [f'Test_{i}_grapheme_root', f'Test_{i}_vowel_diacritic',
f'Test_{i}_consonant_diacritic']
target += [p0[i], p1[i], p2[i]]
submission_df = pd.DataFrame({'row_id': row_id, 'target': target})
 submission_df.to_csv('submission.csv', index=False)
<load_from_csv>
data["LotFrontage"] = data.groupby("Neighborhood")["LotFrontage"].transform(lambda x: x.fillna(x.median()))
train = pd.read_csv(datadir/'train.csv' )<create_dataframe>
for col in('GarageType', 'GarageFinish', 'GarageQual', 'GarageCond'):
data[col] = data[col].fillna('None' )
pred_df = pd.DataFrame({
'grapheme_root': p0,
'vowel_diacritic': p1,
'consonant_diacritic': p2
} )<define_variables>
for col in('GarageYrBlt', 'GarageArea', 'GarageCars'):
data[col] = data[col].fillna(0 )
train_labels = train[['grapheme_root', 'vowel_diacritic', 'consonant_diacritic']].values<set_options>
for col in('BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF','TotalBsmtSF', 'BsmtFullBath', 'BsmtHalfBath'):
data[col] = data[col].fillna(0 )
%matplotlib inline
start_time = time.time()<define_variables>
for col in('BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2'):
data[col] = data[col].fillna('None' )
dataset = '/kaggle/input/bengaliai-cv19'
pretrained = '.. /input/bangla-graphemes-pretrained-weights'<train_model>
data["MasVnrType"] = data["MasVnrType"].fillna("None")
 data["MasVnrArea"] = data["MasVnrArea"].fillna(0 )
if os.path.isfile(os.path.join(pretrained,"GraphemeDenseNet121.h5")) \
and os.path.isfile(os.path.join(pretrained,"hist.csv")) :
print('Model is present')
 else:
print("Error.No Model Found" )<define_variables>
data['MSZoning'] = data['MSZoning'].fillna(data['MSZoning'].mode() [0] )
SIZE = 100
N_ch = 1<load_pretrained>
data = data.drop(['Utilities'], axis=1 )
model = load_model(os.path.join(pretrained, 'GraphemeDenseNet121.h5'))<normalization>
data["Functional"] = data["Functional"].fillna("Typ" )
def resize(df, size=100):
resized = {}
resize_size=100
angle=0
for i in range(df.shape[0]):
image=df.loc[df.index[i]].values.reshape(137,236)
 image_center = tuple(np.array(image.shape[1::-1])/ 2)
 matrix = cv2.getRotationMatrix2D(image_center, angle, 1.0)
 image = cv2.warpAffine(image, matrix, image.shape[1::-1], flags=cv2.INTER_AREA,
borderMode=cv2.BORDER_CONSTANT, borderValue=(0, 0, 0))
matrix = cv2.getRotationMatrix2D(image_center, 0, 1.0)
 image = cv2.warpAffine(image, matrix, image.shape[1::-1], flags=cv2.INTER_AREA,
borderMode=cv2.BORDER_CONSTANT, borderValue=(0, 0, 0))
augBright=A.RandomBrightnessContrast(p=1.0)
 image = augBright(image=image)['image']
_, thresh = cv2.threshold(image, 30, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU)
 contours, _ = cv2.findContours(thresh,cv2.RETR_LIST,cv2.CHAIN_APPROX_SIMPLE)[-2:]
idx = 0
ls_xmin = []
ls_ymin = []
ls_xmax = []
ls_ymax = []
for cnt in contours:
idx += 1
x,y,w,h = cv2.boundingRect(cnt)
 ls_xmin.append(x)
 ls_ymin.append(y)
 ls_xmax.append(x + w)
 ls_ymax.append(y + h)
 xmin = min(ls_xmin)
 ymin = min(ls_ymin)
 xmax = max(ls_xmax)
 ymax = max(ls_ymax)
 roi = image[ymin:ymax,xmin:xmax]
resized_roi = cv2.resize(roi,(resize_size, resize_size),interpolation=cv2.INTER_AREA)
 resized[df.index[i]] = resized_roi.reshape(-1)
 resized_df = pd.DataFrame(resized ).T
return resized_df<define_variables>
data['Electrical'] = data['Electrical'].fillna(data['Electrical'].mode() [0] )
tgt_cols = ['grapheme_root','vowel_diacritic','consonant_diacritic']<predict_on_test>
data['KitchenQual'] = data['KitchenQual'].fillna(data['KitchenQual'].mode() [0] )
row_ids = []
targets = []
id = 0
for i in range(4):
img_df = pd.read_parquet(os.path.join(
 dataset, 'test_image_data_'+str(i)+'.parquet'))
img_df = img_df.drop('image_id', axis = 1)
 img_df = resize(img_df, SIZE)/ 255
X_test = img_df.values.reshape(-1, SIZE, SIZE, N_ch)
 preds = model.predict(X_test)
 for j in range(len(X_test)) :
for k in range(3):
row_ids.append('Test_'+str(id)+'_'+tgt_cols[k])
 targets.append(np.argmax(preds[k][j]))
id += 1<save_to_csv>
data['Exterior1st'] = data['Exterior1st'].fillna(data['Exterior1st'].mode() [0])
 data['Exterior2nd'] = data['Exterior2nd'].fillna(data['Exterior2nd'].mode() [0] )
df_submit = pd.DataFrame({'row_id':row_ids,'target':targets},
columns = ['row_id','target'])
 df_submit.to_csv('submission.csv',index=False)
 df_submit.head(10 )<set_options>
data['SaleType'] = data['SaleType'].fillna(data['SaleType'].mode() [0] )
SEED = 42
LABELS = 'train.csv'
warnings.filterwarnings("ignore")
 fastai.__version__<load_pretrained>
data['MSSubClass'] = data['MSSubClass'].fillna("None" )
learn = load_learner(Path('/kaggle/input/bgraph-rn50-model'),'try2A-rn50-im128-wcv.pkl')
<data_type_conversions>
data['MSSubClass'] = data['MSSubClass'].apply(str)
 data['OverallCond'] = data['OverallCond'].astype(str)
 data['YrSold'] = data['YrSold'].astype(str)
 data['MoSold'] = data['MoSold'].astype(str )
%%time
row_id,target = [],[]
imgnet_mean = [0.485, 0.456, 0.406]
imgnet_std = [0.229, 0.224, 0.225]
for fname in TEST:
df = pd.read_parquet(fname)
 data = 255 - df.iloc[:, 1:].values.reshape(-1, HEIGHT, WIDTH ).astype(np.uint8)
 for idx in range(len(df)) :
name = df.iloc[idx,0]
img =(data[idx]*(255.0/data[idx].max())).astype(np.uint8)
 img = crop_resize(img)
 img = np.stack(( img,)*3,axis=-1)
 img = pil2tensor(img,np.float32 ).div_(255)
 pred, tsor, prob = learn.predict(Image(img))
row_id += [f'{name}_consonant_diacritic',f'{name}_grapheme_root',f'{name}_vowel_diacritic']
c = learn.data.classes
target += [
c[ np.argmax(prob[:7].numpy())].split('_')[1],
c[ np.argmax(prob[7:175].numpy())+ 7 ].split('_')[1],
c[ np.argmax(prob[175:].numpy())+ 175 ].split('_')[1] ]
sub_df = pd.DataFrame({'row_id': row_id, 'target': target})
 sub_df.to_csv('submission.csv', index=False)
 sub_df<import_modules>
cols =('FireplaceQu', 'BsmtQual', 'BsmtCond', 'GarageQual', 'GarageCond',
'ExterQual', 'ExterCond','HeatingQC', 'PoolQC', 'KitchenQual', 'BsmtFinType1',
'BsmtFinType2', 'Functional', 'Fence', 'BsmtExposure', 'GarageFinish', 'LandSlope',
'LotShape', 'PavedDrive', 'Street', 'Alley', 'CentralAir', 'MSSubClass', 'OverallCond',
'YrSold', 'MoSold')
 for c in cols:
lbl = LabelEncoder()
lbl.fit(list(data[c].values))
data[c] = lbl.transform(list(data[c].values))
print('Shape data: {}'.format(data.shape))
import os
import time
import numpy as np
import pandas as pd
from tqdm import tqdm
import math
from sklearn.model_selection import train_test_split
from sklearn import metrics
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.layers import Dense, Input, CuDNNLSTM, Embedding, Dropout, Activation, CuDNNGRU, Conv1D
from keras.layers import Bidirectional, GlobalMaxPool1D
from keras.optimizers import Adam
from keras.models import Model
from keras import backend as K
from keras.engine.topology import Layer
from keras import initializers, regularizers, constraints, optimizers, layers<import_modules>
data = pd.get_dummies(data)
 print(data.shape )
print(device_lib.list_local_devices() )<load_from_csv>
train = data[:1460]
test = data[1460:].drop('SalePrice', axis = 1 )
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 print("Train shape : ",train_df.shape)
 print("Test shape : ",test_df.shape )<split>
X = train.drop('SalePrice', axis = 1)
 y = train['SalePrice']
train_df, val_df = train_test_split(train_df, test_size=0.04, random_state=2018)
 embed_size = 300
max_features = 95000
maxlen = 65
train_X = train_df["question_text"].fillna("_
val_X = val_df["question_text"].fillna("_
test_X = test_df["question_text"].fillna("_
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 val_X = tokenizer.texts_to_sequences(val_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 train_X = pad_sequences(train_X, maxlen=maxlen)
 val_X = pad_sequences(val_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 train_y = train_df['target'].values
val_y = val_df['target'].values<randomize_order>
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state = 42 )
np.random.seed(2018)
 trn_idx = np.random.permutation(len(train_X))
val_idx = np.random.permutation(len(val_X))
train_X = train_X[trn_idx]
val_X = val_X[val_idx]
train_y = train_y[trn_idx]
val_y = val_y[val_idx]<feature_engineering>
xgb = XGBRegressor(booster='gbtree', colsample_bylevel=1,
colsample_bynode=1,colsample_bytree=0.6,
gamma=0, importance_type='gain',
learning_rate=0.01, max_delta_step=0,
max_depth=4,min_child_weight=1.5,
n_estimators=2500, n_jobs=1, nthread=None,
objective='reg:linear', reg_alpha=0.4640,
reg_lambda=0.6, scale_pos_weight=1,
silent=None, subsample=0.8, verbosity=1 )
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8"))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector<choose_model_class>
lgbm = LGBMRegressor(objective='regression',
num_leaves=4,
learning_rate=0.01,
n_estimators=11000,
max_bin=200,
bagging_fraction=0.75,
bagging_freq=5,
bagging_seed=7,
feature_fraction=0.4 )
filter_sizes = [1,2,3,5]
num_filters = 36
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Reshape(( maxlen, embed_size, 1))(x)
 maxpool_pool = []
for i in range(len(filter_sizes)) :
conv = Conv2D(num_filters, kernel_size=(filter_sizes[i], embed_size),
kernel_initializer='he_normal', activation='relu' )(x)
 maxpool_pool.append(MaxPool2D(pool_size=(maxlen - filter_sizes[i] + 1, 1))(conv))
z = Concatenate(axis=1 )(maxpool_pool)
 z = Flatten()(z)
 z = Dropout(0.1 )(z)
 outp = Dense(1, activation="sigmoid" )(z)
 model = Model(inputs=inp, outputs=outp)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'] )<train_model>
gboost = GradientBoostingRegressor(n_estimators=3000,
learning_rate=0.05,
max_depth=4,
max_features='sqrt',
min_samples_leaf=15,
min_samples_split=10,
loss='huber',
random_state =5 )
model.fit(train_X, train_y, batch_size=512, epochs=2, validation_data=(val_X, val_y))<predict_on_test>
xgb.fit(X_train, y_train)
 lgbm.fit(X_train, y_train, eval_metric='rmsle')
 gboost.fit(X_train, y_train )
pred_cnn_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_cnn_val_y>thresh ).astype(int))))<predict_on_test>
pred2 = xgb.predict(X_test)
 pred3 = lgbm.predict(X_test)
 pred4 = gboost.predict(X_test )
pred_cnn_test_y = model.predict([test_X], batch_size=1024, verbose=1 )<set_options>
print('Root Mean Square Logarithmic Error test(XGB)= ' + str(math.sqrt(metrics.mean_squared_log_error(y_test, pred2))))
print('Root Mean Square Logarithmic Error test(LGBM)= ' + str(math.sqrt(metrics.mean_squared_log_error(y_test, pred3))))
print('Root Mean Square Logarithmic Error test(GBoost)= ' + str(math.sqrt(metrics.mean_squared_log_error(y_test, pred4))))
del word_index, embeddings_index, all_embs, embedding_matrix, model, inp, x
time.sleep(10 )<set_options>
lgbm.fit(X, y)
 xgb.fit(X ,y)
 gboost.fit(X, y )
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim<feature_engineering>
prediction_lgbm = np.expm1(lgbm.predict(test))
prediction_xgb = np.expm1(xgb.predict(test))
prediction_gboost = np.expm1(gboost.predict(test))
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8"))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)
 x = Attention(maxlen )(x)
 x = Dense(64, activation="relu" )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer=Adam(lr=1e-3), metrics=['accuracy'] )<train_model>
%%time
model.fit(train_X, train_y, batch_size=512, epochs=5, validation_data=(val_X, val_y))<predict_on_test>
prediction =(prediction_lgbm * 0.38 + prediction_gboost * 0.35 + prediction_xgb * 0.27 )
pred_glove_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_glove_val_y>thresh ).astype(int))))<predict_on_test>
submission = pd.DataFrame({"Id": test.index,"SalePrice": prediction} )
pred_glove_test_y = model.predict([test_X], batch_size=1024, verbose=1 )<set_options>
submission.to_csv('submission.csv', index=False )
<statistical_test><EOS> 
submission.to_csv('submission.csv', index=False )