Kaggle competition: House Prices - Advanced Regression Techniques
pred_val_y, pred_test_y = train_pred(model_lstm_atten(embedding_matrix_3), epochs = 3)
 outputs.append([pred_val_y, pred_test_y, '2 LSTM w/ attention Para'] )<load_from_csv>
train = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 test = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv' )
coefs = [0.20076554,0.07993707,0.11611663,0.14885248,0.15734404,0.17454667,0.14288361]
pred_test_y = np.sum([outputs[i][1]*coefs[i] for i in range(len(coefs)) ], axis = 0)
 pred_test_y =(pred_test_y > 0.34 ).astype(int)
 test_df = pd.read_csv(".. /input/test.csv", usecols=["qid"])
 out_df = pd.DataFrame({"qid":test_df["qid"].values})
 out_df['prediction'] = pred_test_y
out_df.to_csv("submission.csv", index=False )<import_modules>
train[train['GrLivArea'] > 4500].index
import numpy as np
import pandas as pd
import os
import time
from tqdm import tqdm
import matplotlib.pyplot as plt
from matplotlib import style
import seaborn as sns
import tensorflow as tf
from sklearn import metrics
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences
from sklearn.model_selection import train_test_split
from nltk.tokenize import word_tokenize
from scikitplot.metrics import plot_confusion_matrix
from keras.optimizers import Adam
from keras.models import Model
from keras import backend as K
from keras.engine.topology import Layer
from keras import initializers, regularizers, constraints, optimizers, layers
from keras.layers import Add, Bidirectional, CuDNNLSTM, Dense, Input, Embedding,merge, BatchNormalization, Reshape
from keras.models import Model<load_from_csv>
train[train['GarageArea'] > 1220].index
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv")
 print("Train datasets shape:", train.shape)
 print("Test datasets shape:", test.shape )<split>
train[train['TotalBsmtSF'] > 5000].index
train_data, validation_data = train_test_split(train, test_size=.1, random_state=1234)
 max_length = 100
max_features = 50000
token = Tokenizer(num_words=max_features)
 token.fit_on_texts(list(np.asarray(train_data.question_text)))
 xtrain = token.texts_to_sequences(np.asarray(train_data.question_text))
xvalidate = token.texts_to_sequences(np.asarray(validation_data.question_text))
xtest = token.texts_to_sequences(np.asarray(test.question_text))
xtrain = pad_sequences(xtrain, maxlen=max_length)
 xvalidate = pad_sequences(xvalidate, maxlen=max_length)
 xtest = pad_sequences(xtest, maxlen=max_length)
 ytrain = train_data.target
yvaliate = validation_data.target<set_options>
train[train['TotalBsmtSF'] > 4000].index
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim<statistical_test>
train = train.drop([523, 581, 1061, 1190, 1298] )
em_file = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word, *arr):
return word, np.asarray(arr, dtype='float32')
 embedding_index = dict(get_coefs(*d.split(' ')) for d in open(em_file))
all_embs = np.stack(embedding_index.values())
 em_mean, em_std = all_embs.mean() , all_embs.std()
em_size = all_embs.shape[1]
word_index = token.word_index
nb_words = min(max_features, len(word_index))
em_matrix = np.random.normal(em_mean, em_std,(nb_words, em_size))
for word, i in word_index.items() :
if i >= max_features: continue
em_v = embedding_index.get(word)
 if em_v is not None:
em_matrix[i] = em_v 
<init_hyperparams>
train.reset_index(drop=True, inplace=True )
atten_layers = 1
num_att = 10
lstm_units = 128<choose_model_class>
y_train = train['SalePrice'].reset_index(drop=True)
 train = train.drop(['SalePrice'], axis=1 )
def attention_model(em_matrix, atten_layers=atten_layers, num_att=num_att, lstm_units=lstm_units):
inp = Input(shape=(max_length,))
x = Embedding(max_features, em_matrix.shape[1], weights=[em_matrix], trainable=False )(inp)
 x = Bidirectional(CuDNNLSTM(lstm_units, return_sequences=True))(x)
 def attention_block(layer, atten_len=max_length, num_multi=num_att, dense_units=lstm_units*2):
atten_list = []
for _ in range(num_multi):
atten_list.append(Attention(atten_len )(layer))
add_layer = Add()(atten_list)
 add_layer = BatchNormalization()(add_layer)
 dense_layer = Dense(dense_units, activation='relu' )(add_layer)
 return Add()([add_layer, dense_layer])
 for j in range(atten_layers):
if j ==0:
x = attention_block(x)
 else:
x = Reshape(( lstm_units*2, 1))(x)
 x = attention_block(x, atten_len=lstm_units*2)
 x = Dense(64, activation='relu' )(x)
 out = Dense(1, activation='sigmoid' )(x)
 model = Model(inp, out)
 model.summary()
model.compile(loss='binary_crossentropy', metrics=['accuracy'], optimizer='adam')
 return model<choose_model_class>
train.drop(['Id'], axis=1, inplace=True)
 test.drop(['Id'], axis=1, inplace=True )
def att_new_model(em_matrix, atten_layers=atten_layers, num_att=num_att, lstm_units=lstm_units):
inp = Input(shape=(max_length,))
x = Embedding(max_features, em_matrix.shape[1], weights=[em_matrix], trainable=False )(inp)
 x = Bidirectional(CuDNNLSTM(lstm_units, return_sequences=True))(x)
 def attention_block(layer, atten_len=max_length, num_multi=num_att, dense_units=lstm_units*2):
add_layer_list = []
for j in range(num_att):
atten_list = list()
for _ in range(num_multi):
atten_list.append(Attention(atten_len )(layer))
add_layer_list.append(Add()(atten_list))
add_layer = concatenate(add_layer_list)
 add_layer = BatchNormalization()(add_layer)
 dense_layer = Dense(dense_units, activation='relu' )(add_layer)
 return dense_layer
x = attention_block(x)
 x = Dense(64, activation='relu' )(x)
 out = Dense(1, activation='sigmoid' )(x)
 model = Model(inp, out)
 model.summary()
model.compile(loss='binary_crossentropy', metrics=['accuracy'], optimizer='adam')
 return model<predict_on_test>
all_features = pd.concat(( train,test)).reset_index(drop=True)
 all_features.shape
model = attention_model(em_matrix=em_matrix, atten_layers=1, lstm_units=64, num_att=10)
 model.fit(xtrain, ytrain, epochs=2, batch_size=512, validation_data=(xvalidate, yvaliate), verbose=1)
 pred_vali_glove = model.predict(xvalidate, batch_size=1024)
 best_thre = evaluate(yvaliate, pred_vali_glove)
 pred_test_glove = model.predict(xtest )<set_options>
total = all_features.isnull().sum().sort_values(ascending=False)
 percent =(all_features.isnull().sum() /all_features.isnull().count() ).sort_values(ascending=False)
 missing_data = pd.concat([total,percent], axis=1, keys=['Total','Percent'])
 missing_data.head(40 )
del embedding_index, all_embs, word_index, em_matrix
gc.collect()
time.sleep(10 )<statistical_test>
all_features['MSSubClass'] = all_features['MSSubClass'].apply(str)
 all_features['YrSold'] = all_features['YrSold'].apply(str)
 all_features['MoSold'] = all_features['MoSold'].apply(str )
em_file = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word, *arr):
return word, np.asarray(arr, dtype='float32')
 embedding_index = dict(get_coefs(*o.split(" ")) for o in open(em_file, encoding="utf8", errors='ignore')if len(o)>100)
 all_embs = np.stack(embedding_index.values())
 em_mean, em_std = all_embs.mean() , all_embs.std()
em_size = all_embs.shape[1]
word_index = token.word_index
nb_words = min(max_features, len(word_index))
em_matrix = np.random.normal(em_mean, em_std,(nb_words, em_size))
for word, i in word_index.items() :
if i >= max_features: continue
em_v = embedding_index.get(word)
 if em_v is not None:
em_matrix[i] = em_v 
<predict_on_test>
common_vars = ['Exterior1st','Exterior2nd','SaleType','Electrical','KitchenQual']
for var in common_vars:
all_features[var] = all_features[var].fillna(all_features[var].mode() [0])
 all_features['MSZoning'] = all_features.groupby('MSSubClass')['MSZoning'].transform(lambda x: x.fillna(x.mode() [0]))
all_features['Functional'] = all_features['Functional'].fillna('Typ' )
model = attention_model(em_matrix=em_matrix)
 model.fit(xtrain, ytrain, epochs=2, batch_size=512, validation_data=(xvalidate, yvaliate), verbose=1)
 pred_vali_para = model.predict(xvalidate, batch_size=1024)
 best_thre = evaluate(yvaliate, pred_vali_para)
 pred_test_para = model.predict(xtest )<drop_column>
col_str = ['GarageType', 'GarageFinish', 'GarageQual', 'GarageCond','BsmtQual',
'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2',"PoolQC"
,'Alley','Fence','MiscFeature','FireplaceQu','MasVnrType','Utilities']
for col in col_str:
all_features[col] = all_features[col].fillna('None' )
del embedding_index, all_embs, word_index, em_matrix
gc.collect()
time.sleep(10 )<train_model>
col_num = ['GarageYrBlt','GarageArea','GarageCars','MasVnrArea','BsmtFinSF1','BsmtFinSF2','BsmtFullBath','BsmtHalfBath','FullBath','HalfBath','BsmtUnfSF','TotalBsmtSF']
for col in col_num:
all_features[col] = all_features[col].fillna(0)
 all_features['LotFrontage'] = all_features.groupby('Neighborhood')['LotFrontage'].transform(lambda x: x.fillna(x.median()))
lr = LinearRegression()
model_num = 2
pred_data = np.empty([len(pred_vali_para), model_num])
 pred_data[:, 0] = pred_vali_glove.reshape(-1,)
 pred_data[:, 1] = pred_vali_para.reshape(-1,)
 lr.fit(pred_data, yvaliate)
 weights = lr.coef_
sub_pred_weighted = np.sum([pred_data[:, i]*weights[i] for i in range(model_num)], axis=0)
 best_thre = evaluate(yvaliate ,sub_pred_weighted )<prepare_output>
all_features.isnull().sum().sort_values(ascending=False ).head(5 )
sub_data = np.empty([len(xtest), model_num])
 sub_data[:, 0] = pred_test_glove.reshape(-1,)
 sub_data[:, 1] = pred_test_para.reshape(-1, )<save_to_csv>
skewness = all_features[num_features].apply(lambda x: skew(x)).sort_values(ascending=False)
 high_skewness = skewness[abs(skewness)> 0.5]
print("There are {} numerical features with Skew > 0.5 :".format(high_skewness.shape[0]))
high_skewness.sort_values(ascending=False )
sub_pred = np.sum([sub_data[:, i]*weights[i] for i in range(model_num)], axis=0)
 sub_pred =(sub_pred > best_thre ).astype(int)
 sub_df = pd.DataFrame({'qid':test.qid.values})
 sub_df['prediction'] = sub_pred
sub_df.to_csv('submission.csv', index=False)
<import_modules>
skewed_features = high_skewness.index
for feat in skewed_features:
all_features[feat] = boxcox1p(all_features[feat], boxcox_normmax(all_features[feat] + 1))
import os
import time
import numpy as np
import pandas as pd
from tqdm import tqdm
import math
from sklearn.model_selection import train_test_split
from sklearn import metrics
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.layers import Dense, Input, LSTM, Embedding, Dropout, Activation, CuDNNGRU, Conv1D
from keras.layers import Bidirectional, GlobalMaxPool1D
from keras import regularizers
from keras.models import Model
from keras import initializers, regularizers, constraints, optimizers, layers<load_from_csv>
new_skewness = all_features[num_features].apply(lambda x: skew(x)).sort_values(ascending=False)
 new_high_skewness = new_skewness[abs(new_skewness)> 0.5]
print("There are {} skewed numerical features after Box Cox transform".format(new_high_skewness.shape[0]))
print("Mean skewnees: {}".format(np.mean(new_high_skewness)))
 new_high_skewness.sort_values(ascending=False )
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 print("Train shape : ",train_df.shape)
 print("Test shape : ",test_df.shape )<split>
all_features['TotalSF']=all_features['TotalBsmtSF'] + all_features['1stFlrSF'] + all_features['2ndFlrSF']
all_features['Total_Bathrooms'] =(all_features['FullBath'] +(0.5 * all_features['HalfBath'])+
all_features['BsmtFullBath'] +(0.5 * all_features['BsmtHalfBath']))
all_features['Total_porch_sf'] =(all_features['OpenPorchSF'] + all_features['3SsnPorch'] +
all_features['EnclosedPorch'] + all_features['ScreenPorch'] +
all_features['WoodDeckSF'])
 all_features['haspool'] = all_features['PoolArea'].apply(lambda x: 1 if x > 0 else 0)
 all_features['hasgarage'] = all_features['GarageArea'].apply(lambda x: 1 if x > 0 else 0)
 all_features['hasbsmt'] = all_features['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 all_features['hasfireplace'] = all_features['Fireplaces'].apply(lambda x: 1 if x > 0 else 0)
 all_features = all_features.drop(['MasVnrArea', 'OpenPorchSF', 'WoodDeckSF', 'BsmtFinSF1','2ndFlrSF'], axis=1 )
train_df, val_df = train_test_split(train_df, test_size=0.1, random_state=2018)
 embed_size = 300
max_features = 50000
maxlen = 100
train_X = train_df["question_text"].fillna("_na_" ).values
val_X = val_df["question_text"].fillna("_na_" ).values
test_X = test_df["question_text"].fillna("_na_" ).values
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 val_X = tokenizer.texts_to_sequences(val_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 train_X = pad_sequences(train_X, maxlen=maxlen)
 val_X = pad_sequences(val_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 train_y = train_df['target'].values
val_y = val_df['target'].values<statistical_test>
all_features = pd.get_dummies(all_features ).reset_index(drop=True)
 all_features.shape
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
compressive_mat = np.random.randn(embed_size // 3 , all_embs.shape[1])
 word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix1 = np.random.normal(emb_mean, emb_std,(nb_words, embed_size // 3))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix1[i] = compressive_mat.dot(embedding_vector)
 del embeddings_index
time.sleep(10 )<statistical_test>
overfit = []
for i in X.columns:
counts = X[i].value_counts()
zeros = counts.iloc[0]
if zeros / len(X)* 100 > 99.95:
overfit.append(i)
 overfit = list(overfit)
 overfit.append('MSZoning_C(all)')
 X = X.drop(overfit, axis=1 ).copy()
X_test = X_test.drop(overfit, axis=1 ).copy()
print(X.shape,y_train.shape,X_test.shape )
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE,
encoding="utf8", errors='ignore'))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
compressive_mat = np.random.randn(embed_size // 3, all_embs.shape[1])
 word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix2 = np.random.normal(emb_mean, emb_std,(nb_words, embed_size // 3))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix2[i] = compressive_mat.dot(embedding_vector)
 del embeddings_index
time.sleep(10 )<statistical_test>
from datetime import datetime
from sklearn.preprocessing import RobustScaler
from sklearn.model_selection import KFold, cross_val_score
from sklearn.metrics import mean_squared_error, make_scorer
from sklearn.linear_model import ElasticNetCV, LassoCV, RidgeCV
from sklearn.pipeline import make_pipeline
from sklearn.linear_model import LinearRegression
from sklearn.ensemble import GradientBoostingRegressor
from sklearn.svm import SVR
from mlxtend.regressor import StackingCVRegressor
from sklearn.linear_model import LinearRegression
from xgboost import XGBRegressor
from lightgbm import LGBMRegressor
EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
compressive_mat = np.random.randn(embed_size // 3, all_embs.shape[1])
 word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix3 = np.random.normal(emb_mean, emb_std,(nb_words, embed_size // 3))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix3[i] = compressive_mat.dot(embedding_vector)
 del embeddings_index
time.sleep(10 )<concatenate>
kfolds = KFold(n_splits=16, shuffle=True, random_state=42)
 def cv_rmse(model, X=X):
rmse = np.sqrt(-cross_val_score(model, X, y_train, scoring="neg_mean_squared_error", cv=kfolds))
return(rmse)
 def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))
embedding_matrix = np.concatenate([embedding_matrix1, embedding_matrix2, embedding_matrix3],axis=1)
<choose_model_class>
lightgbm = LGBMRegressor(objective='regression',
num_leaves=4,
learning_rate=0.01,
n_estimators=9000,
max_bin=200,
bagging_fraction=0.75,
bagging_freq=5,
bagging_seed=7,
feature_fraction=0.2,
feature_fraction_seed=7,
min_sum_hessian_in_leaf = 11,
verbose=-1,
random_state=42)


 ridge_alpha = [14.5, 14.6, 14.7, 14.8, 14.9, 15, 15.1, 15.2, 15.3, 15.4, 15.5]
lasso_alpha = [5e-05, 0.0001, 0.0002, 0.0003, 0.0004, 0.0005, 0.0006, 0.0007, 0.0008]
elastic_alpha = [0.0001, 0.0002, 0.0003, 0.0004, 0.0005, 0.0006, 0.0007]
e_l1ratio = [0.8, 0.85, 0.9, 0.95, 0.99, 1]
ridge = make_pipeline(RobustScaler() , RidgeCV(alphas=ridge_alpha, cv=kfolds))
lasso = make_pipeline(RobustScaler() , LassoCV(max_iter=1e7,
alphas=lasso_alpha,random_state=42, cv=kfolds))
elasticnet = make_pipeline(RobustScaler() , ElasticNetCV(max_iter=1e7,
alphas=elastic_alpha, cv=kfolds, l1_ratio=e_l1ratio))
svr = make_pipeline(RobustScaler() , SVR(C= 20, epsilon= 0.008, gamma=0.0003))
stack_gen = StackingCVRegressor(regressors=(ridge, lasso, elasticnet, svr, lightgbm),
meta_regressor=elasticnet,
use_features_in_secondary=True )
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=True )(inp)
 x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)
 x = GlobalMaxPool1D()(x)
 x = Dense(16, activation="relu" )(x)
 x = Dropout(0.1 )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 print(model.summary() )<train_model>
scores = {}
score = cv_rmse(lightgbm)
 print("lightgbm: {:.4f}({:.4f})".format(score.mean() , score.std()))
scores['lightgbm'] =(score.mean() , score.std() )
model.fit(train_X, train_y, batch_size=512, epochs=2, validation_data=(val_X, val_y))<find_best_params>
score = cv_rmse(ridge)
 print("ridge: {:.4f}({:.4f})".format(score.mean() , score.std()))
scores['ridge'] =(score.mean() , score.std() )
pred_glove_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 f_scores = []
for thresh in np.arange(0.1, 0.9, 0.01):
thresh = np.round(thresh, 2)
 f_score = metrics.f1_score(val_y,(pred_glove_val_y>thresh ).astype(int))
f_scores.append(( f_score, thresh))
fmax, opt_thresh = max(f_scores)
 print("F1 score at threshold {0} is {1}".format(opt_thresh, fmax)) 
<predict_on_test>
score = cv_rmse(lasso)
 print("lasso: {:.4f}({:.4f})".format(score.mean() , score.std()))
scores['lasso'] =(score.mean() , score.std() )
pred_all_test_y = model.predict([test_X], batch_size=1024, verbose=1 )<save_to_csv>
score = cv_rmse(elasticnet)
 print("elasticnet: {:.4f}({:.4f})".format(score.mean() , score.std()))
scores['elasticnet'] =(score.mean() , score.std() )
pred_test_y = pred_all_test_y
pred_test_y =(pred_test_y>opt_thresh ).astype(int)
 out_df = pd.DataFrame({"qid":test_df["qid"].values})
 out_df['prediction'] = pred_test_y
out_df.to_csv("submission.csv", index=False )<import_modules>
score = cv_rmse(svr)
 print("svr: {:.4f}({:.4f})".format(score.mean() , score.std()))
scores['svr'] =(score.mean() , score.std() )
import nltk, re, string
from nltk.corpus import stopwords
from nltk.stem import SnowballStemmer<categorify>
print('----START Fit----',datetime.now())
 print('Elasticnet')
 elastic_model = elasticnet.fit(X, y_train)
 print('Lasso')
 lasso_model = lasso.fit(X, y_train)
 print('Ridge')
 ridge_model = ridge.fit(X, y_train)
 print('lightgbm')
 lgb_model = lightgbm.fit(X, y_train)
 print('svr')
 svr_model = svr.fit(X, y_train)
 print('stack_gen')
 stack_gen_model = stack_gen.fit(np.array(X), np.array(y_train))
def clean_text(text):
print(text)
 text = text.translate(string.punctuation)
 text = text.lower()
text = re.sub(r"[^A-Za-z0-9^,!.\/'+-=]", " ", text)
 text = re.sub(r"what's", "what is ", text)
 text = re.sub(r"'s", " ", text)
 text = re.sub(r"'ve", " have ", text)
 text = re.sub(r"n't", " not ", text)
 text = re.sub(r"i'm", "i am ", text)
 text = re.sub(r"'re", " are ", text)
 text = re.sub(r"'d", " would ", text)
 text = re.sub(r"'ll", " will ", text)
 text = re.sub(r",", " ", text)
 text = re.sub(r"\.", " ", text)
 text = re.sub(r"!", " ! ", text)
 text = re.sub(r"\/", " ", text)
 text = re.sub(r"\^", " ^ ", text)
 text = re.sub(r"\+", " + ", text)
 text = re.sub(r"\-", " - ", text)
 text = re.sub(r"\=", " = ", text)
 text = re.sub(r"'", " ", text)
 text = re.sub(r"(\d+ )(k)", r"\g<1>000", text)
 text = re.sub(r":", " : ", text)
 text = re.sub(r" e g ", " eg ", text)
 text = re.sub(r" b g ", " bg ", text)
 text = re.sub(r" u s ", " american ", text)
 text = re.sub(r"\0s", "0", text)
 text = re.sub(r" 9 11 ", "911", text)
 text = re.sub(r"e - mail", "email", text)
 text = re.sub(r"j k", "jk", text)
 text = re.sub(r"\s{2,}", " ", text)
 print(text)
 print("")
 return text<load_from_csv>
def blend_predictions(X):
return(( 0.12 * elastic_model.predict(X)) + \ 
(0.12 * lasso_model.predict(X)) + \ 
(0.12 * ridge_model.predict(X)) + \ 
(0.22 * lgb_model.predict(X)) + \ 
(0.1 * svr_model.predict(X)) + \ 
(0.32 * stack_gen_model.predict(np.array(X))))
train_df = pd.read_csv('.. /input/train.csv')
 test_df = pd.read_csv('.. /input/test.csv')
<prepare_x_and_y>
blended_score = rmsle(y_train, blend_predictions(X))
scores['blended'] =(blended_score, 0)
 print('RMSLE score on train data:')
 print(blended_score )
X_train = train_df["question_text"].fillna("na" ).values
X_test = test_df["question_text"].fillna("na" ).values
y = train_df["target"]<import_modules>
submission = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/sample_submission.csv")
 submission.iloc[:,1] = np.floor(np.expm1(blend_predictions(X_test)) )
from keras.models import Model
from keras.layers import Input, Dense, Embedding, concatenate
from keras.layers import CuDNNGRU, Bidirectional, GlobalAveragePooling1D, GlobalMaxPooling1D, Conv1D
from keras.layers import Add, BatchNormalization, Activation, CuDNNLSTM, Dropout
from keras.layers import *
from keras.models import *
from keras.preprocessing import text, sequence
from keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau
import gc
from sklearn import metrics 
<string_transform>
submission.to_csv("submission.csv", index=False )
<categorify><EOS> 
submission.to_csv("submission.csv", index=False )