Kaggle competition: Quora Insincere Questions Classification
<SOS> metric: FScore_1 Kaggle data source: quora-insincere-questions-classification<feature_engineering>
tqdm.pandas()
col_num = ['GarageYrBlt','GarageArea','GarageCars','MasVnrArea','BsmtFinSF1','BsmtFinSF2','BsmtFullBath','BsmtHalfBath','FullBath','HalfBath','BsmtUnfSF','TotalBsmtSF']
for col in col_num:
all_features[col] = all_features[col].fillna(0)
 all_features['LotFrontage'] = all_features.groupby('Neighborhood')['LotFrontage'].transform(lambda x: x.fillna(x.median()))<sort_values>
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv")
 print("Train shape : ", train.shape)
 print("Test shape : ", test.shape )
all_features.isnull().sum().sort_values(ascending=False ).head(5 )<sort_values>
train["question_text"] = train["question_text"].str.lower()
test["question_text"] = test["question_text"].str.lower()
puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…',
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─',
'▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]
def clean_text(x):
x = str(x)
 for punct in puncts:
x = x.replace(punct, f' {punct} ')
 return x
train["question_text"] = train["question_text"].apply(lambda x: clean_text(x))
test["question_text"] = test["question_text"].apply(lambda x: clean_text(x))
skewness = all_features[num_features].apply(lambda x: skew(x)).sort_values(ascending=False)
 high_skewness = skewness[abs(skewness)> 0.5]
print("There are {} numerical features with Skew > 0.5 :".format(high_skewness.shape[0]))
high_skewness.sort_values(ascending=False )<feature_engineering>
embed_size = 300
max_features = None
maxlen = 72
X = train["question_text"].fillna("_na_" ).values
X_test = test["question_text"].fillna("_na_" ).values
tokenizer = Tokenizer(num_words=max_features, filters='')
 tokenizer.fit_on_texts(list(X))
X = tokenizer.texts_to_sequences(X)
 X_test = tokenizer.texts_to_sequences(X_test)
 X = pad_sequences(X, maxlen=maxlen)
 X_test = pad_sequences(X_test, maxlen=maxlen)
 Y = train['target'].values
sub = test[['qid']]
skewed_features = high_skewness.index
for feat in skewed_features:
all_features[feat] = boxcox1p(all_features[feat], boxcox_normmax(all_features[feat] + 1))<sort_values>
del train, test
gc.collect()
new_skewness = all_features[num_features].apply(lambda x: skew(x)).sort_values(ascending=False)
 new_high_skewness = new_skewness[abs(new_skewness)> 0.5]
print("There are {} skewed numerical features after Box Cox transform".format(new_high_skewness.shape[0]))
print("Mean skewnees: {}".format(np.mean(new_high_skewness)))
 new_high_skewness.sort_values(ascending=False )<feature_engineering>
word_index = tokenizer.word_index
max_features = len(word_index)+1
def load_glove(word_index):
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if o.split(" ")[0] in word_index)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
embedding_matrix = np.random.normal(emb_mean, emb_std,(max_features, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_fasttext(word_index):
EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100 and o.split(" ")[0] in word_index)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
embedding_matrix = np.random.normal(emb_mean, emb_std,(max_features, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_para(word_index):
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100 and o.split(" ")[0] in word_index)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
embedding_matrix = np.random.normal(emb_mean, emb_std,(max_features, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
all_features['TotalSF']=all_features['TotalBsmtSF'] + all_features['1stFlrSF'] + all_features['2ndFlrSF']
all_features['Total_Bathrooms'] =(all_features['FullBath'] +(0.5 * all_features['HalfBath'])+
all_features['BsmtFullBath'] +(0.5 * all_features['BsmtHalfBath']))
all_features['Total_porch_sf'] =(all_features['OpenPorchSF'] + all_features['3SsnPorch'] +
all_features['EnclosedPorch'] + all_features['ScreenPorch'] +
all_features['WoodDeckSF'])
 all_features['haspool'] = all_features['PoolArea'].apply(lambda x: 1 if x > 0 else 0)
 all_features['hasgarage'] = all_features['GarageArea'].apply(lambda x: 1 if x > 0 else 0)
 all_features['hasbsmt'] = all_features['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 all_features['hasfireplace'] = all_features['Fireplaces'].apply(lambda x: 1 if x > 0 else 0)
 all_features = all_features.drop(['MasVnrArea', 'OpenPorchSF', 'WoodDeckSF', 'BsmtFinSF1','2ndFlrSF'], axis=1 )<categorify>
embedding_matrix_1 = load_glove(word_index)
 embedding_matrix_3 = load_para(word_index)
 embedding_matrix = np.mean(( embedding_matrix_1, embedding_matrix_3), axis=0)
 np.shape(embedding_matrix )
all_features = pd.get_dummies(all_features ).reset_index(drop=True)
 all_features.shape<train_model>
def f1(y_true, y_pred):
def recall(y_true, y_pred):
true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
 recall = true_positives /(possible_positives + K.epsilon())
 return recall
def precision(y_true, y_pred):
true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
 precision = true_positives /(predicted_positives + K.epsilon())
 return precision
precision = precision(y_true, y_pred)
 recall = recall(y_true, y_pred)
 return 2*(( precision*recall)/(precision+recall+K.epsilon()))
def f1_smart(y_true, y_pred):
args = np.argsort(y_pred)
 tp = y_true.sum()
fs =(tp - np.cumsum(y_true[args[:-1]])) / np.arange(y_true.shape[0] + tp - 1, tp, -1)
 res_idx = np.argmax(fs)
 return 2 * fs[res_idx],(y_pred[args[res_idx]] + y_pred[args[res_idx + 1]])/ 2
overfit = []
for i in X.columns:
counts = X[i].value_counts()
zeros = counts.iloc[0]
if zeros / len(X)* 100 > 99.95:
overfit.append(i)
 overfit = list(overfit)
 overfit.append('MSZoning_C(all)')
 X = X.drop(overfit, axis=1 ).copy()
X_test = X_test.drop(overfit, axis=1 ).copy()
print(X.shape,y_train.shape,X_test.shape )<import_modules>
def squash(x, axis=-1):
s_squared_norm = K.sum(K.square(x), axis, keepdims=True)
 scale = K.sqrt(s_squared_norm + K.epsilon())
 return x / scale
class Capsule(Layer):
def __init__(self, num_capsule, dim_capsule, routings=3, kernel_size=(9, 1), share_weights=True,
activation='default', **kwargs):
super(Capsule, self ).__init__(**kwargs)
 self.num_capsule = num_capsule
self.dim_capsule = dim_capsule
self.routings = routings
self.kernel_size = kernel_size
self.share_weights = share_weights
if activation == 'default':
self.activation = squash
else:
self.activation = Activation(activation)
 def build(self, input_shape):
super(Capsule, self ).build(input_shape)
 input_dim_capsule = input_shape[-1]
if self.share_weights:
self.W = self.add_weight(name='capsule_kernel',
shape=(1, input_dim_capsule,
self.num_capsule * self.dim_capsule),
initializer='glorot_uniform',
trainable=True)
 else:
input_num_capsule = input_shape[-2]
self.W = self.add_weight(name='capsule_kernel',
shape=(input_num_capsule,
input_dim_capsule,
self.num_capsule * self.dim_capsule),
initializer='glorot_uniform',
trainable=True)
 def call(self, u_vecs):
if self.share_weights:
u_hat_vecs = K.conv1d(u_vecs, self.W)
 else:
u_hat_vecs = K.local_conv1d(u_vecs, self.W, [1], [1])
 batch_size = K.shape(u_vecs)[0]
input_num_capsule = K.shape(u_vecs)[1]
u_hat_vecs = K.reshape(u_hat_vecs,(batch_size, input_num_capsule,
self.num_capsule, self.dim_capsule))
u_hat_vecs = K.permute_dimensions(u_hat_vecs,(0, 2, 1, 3))
b = K.zeros_like(u_hat_vecs[:, :, :, 0])
 for i in range(self.routings):
b = K.permute_dimensions(b,(0, 2, 1))
c = K.softmax(b)
 c = K.permute_dimensions(c,(0, 2, 1))
b = K.permute_dimensions(b,(0, 2, 1))
outputs = self.activation(tf.keras.backend.batch_dot(c, u_hat_vecs, [2, 2]))
if i < self.routings - 1:
b = tf.keras.backend.batch_dot(outputs, u_hat_vecs, [2, 3])
 return outputs
def compute_output_shape(self, input_shape):
return(None, self.num_capsule, self.dim_capsule )
from datetime import datetime
from sklearn.preprocessing import RobustScaler
from sklearn.model_selection import KFold, cross_val_score
from sklearn.metrics import mean_squared_error, make_scorer
from sklearn.linear_model import ElasticNetCV, LassoCV, RidgeCV
from sklearn.pipeline import make_pipeline
from sklearn.linear_model import LinearRegression
from sklearn.ensemble import GradientBoostingRegressor
from sklearn.svm import SVR
from mlxtend.regressor import StackingCVRegressor
from sklearn.linear_model import LinearRegression
from xgboost import XGBRegressor
from lightgbm import LGBMRegressor<compute_train_metric>
def create_capsule(embedding_matrix):
K.clear_session()
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = SpatialDropout1D(rate=0.2 )(x)
 x = Bidirectional(CuDNNGRU(100, return_sequences=True,
kernel_initializer=glorot_normal(seed=12300), recurrent_initializer=orthogonal(gain=1.0, seed=10000)) )(x)
 x = Capsule(num_capsule=10, dim_capsule=10, routings=4, share_weights=True )(x)
 x = Flatten()(x)
 x = Dense(100, activation="relu", kernel_initializer=glorot_normal(seed=12300))(x)
 x = Dropout(0.5 )(x)
 x = BatchNormalization()(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer=Adam() ,)
 return model
kfolds = KFold(n_splits=16, shuffle=True, random_state=42)
 def cv_rmse(model, X=X):
rmse = np.sqrt(-cross_val_score(model, X, y_train, scoring="neg_mean_squared_error", cv=kfolds))
return(rmse)
 def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))<choose_model_class>
class CyclicLR(Callback):
def __init__(self, base_lr=0.001, max_lr=0.006, step_size=2000., mode='triangular',
gamma=1., scale_fn=None, scale_mode='cycle'):
super(CyclicLR, self ).__init__()
self.base_lr = base_lr
self.max_lr = max_lr
self.step_size = step_size
self.mode = mode
self.gamma = gamma
if scale_fn == None:
if self.mode == 'triangular':
self.scale_fn = lambda x: 1.
 self.scale_mode = 'cycle'
elif self.mode == 'triangular2':
self.scale_fn = lambda x: 1/(2.**(x-1))
self.scale_mode = 'cycle'
elif self.mode == 'exp_range':
self.scale_fn = lambda x: gamma**(x)
 self.scale_mode = 'iterations'
else:
self.scale_fn = scale_fn
self.scale_mode = scale_mode
self.clr_iterations = 0.
 self.trn_iterations = 0.
 self.history = {}
self._reset()
def _reset(self, new_base_lr=None, new_max_lr=None,
new_step_size=None):

 if new_base_lr != None:
self.base_lr = new_base_lr
if new_max_lr != None:
self.max_lr = new_max_lr
if new_step_size != None:
self.step_size = new_step_size
self.clr_iterations = 0.
 def clr(self):
cycle = np.floor(1+self.clr_iterations/(2*self.step_size))
x = np.abs(self.clr_iterations/self.step_size - 2*cycle + 1)
 if self.scale_mode == 'cycle':
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(cycle)
 else:
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(self.clr_iterations)
 def on_train_begin(self, logs={}):
logs = logs or {}
if self.clr_iterations == 0:
K.set_value(self.model.optimizer.lr, self.base_lr)
 else:
K.set_value(self.model.optimizer.lr, self.clr())
 def on_batch_end(self, epoch, logs=None):
logs = logs or {}
self.trn_iterations += 1
self.clr_iterations += 1
self.history.setdefault('lr', [] ).append(K.get_value(self.model.optimizer.lr))
self.history.setdefault('iterations', [] ).append(self.trn_iterations)
 for k, v in logs.items() :
self.history.setdefault(k, [] ).append(v)
 K.set_value(self.model.optimizer.lr, self.clr())
lightgbm = LGBMRegressor(objective='regression',
num_leaves=4,
learning_rate=0.01,
n_estimators=9000,
max_bin=200,
bagging_fraction=0.75,
bagging_freq=5,
bagging_seed=7,
feature_fraction=0.2,
feature_fraction_seed=7,
min_sum_hessian_in_leaf = 11,
verbose=-1,
random_state=42)


 ridge_alpha = [14.5, 14.6, 14.7, 14.8, 14.9, 15, 15.1, 15.2, 15.3, 15.4, 15.5]
lasso_alpha = [5e-05, 0.0001, 0.0002, 0.0003, 0.0004, 0.0005, 0.0006, 0.0007, 0.0008]
elastic_alpha = [0.0001, 0.0002, 0.0003, 0.0004, 0.0005, 0.0006, 0.0007]
e_l1ratio = [0.8, 0.85, 0.9, 0.95, 0.99, 1]
ridge = make_pipeline(RobustScaler() , RidgeCV(alphas=ridge_alpha, cv=kfolds))
lasso = make_pipeline(RobustScaler() , LassoCV(max_iter=1e7,
alphas=lasso_alpha,random_state=42, cv=kfolds))
elasticnet = make_pipeline(RobustScaler() , ElasticNetCV(max_iter=1e7,
alphas=elastic_alpha, cv=kfolds, l1_ratio=e_l1ratio))
svr = make_pipeline(RobustScaler() , SVR(C= 20, epsilon= 0.008, gamma=0.0003))
stack_gen = StackingCVRegressor(regressors=(ridge, lasso, elasticnet, svr, lightgbm),
meta_regressor=elasticnet,
use_features_in_secondary=True )<compute_test_metric>
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim
scores = {}
score = cv_rmse(lightgbm)
 print("lightgbm: {:.4f}({:.4f})".format(score.mean() , score.std()))
scores['lightgbm'] =(score.mean() , score.std() )<compute_test_metric>
def create_lstm_atten(embedding_matrix):
K.clear_session()
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = SpatialDropout1D(0.1 )(x)
 x = Bidirectional(CuDNNLSTM(40, return_sequences=True))(x)
 y = Bidirectional(CuDNNGRU(40, return_sequences=True))(x)
 atten_1 = Attention(maxlen )(x)
 atten_2 = Attention(maxlen )(y)
 avg_pool = GlobalAveragePooling1D()(y)
 max_pool = GlobalMaxPooling1D()(y)
 conc = concatenate([atten_1, atten_2, avg_pool, max_pool])
 conc = Dense(16, activation="relu" )(conc)
 conc = Dropout(0.1 )(conc)
 outp = Dense(1, activation="sigmoid" )(conc)
 model = Model(inputs=inp, outputs=outp)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=[f1])
 return model
score = cv_rmse(ridge)
 print("ridge: {:.4f}({:.4f})".format(score.mean() , score.std()))
scores['ridge'] =(score.mean() , score.std() )<compute_test_metric>
def create_cnn(embedding_matrix):
K.clear_session()
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = SpatialDropout1D(rate=0.2 )(x)
 conv1 = Conv1D(400, 4, activation = 'relu' )(x)
 conv1 = GlobalMaxPool1D()(conv1)
 out = Dropout(0.5 )(conv1)
 out = Dense(400, activation="relu", kernel_initializer=glorot_normal(seed=12300))(out)
 out = Dense(1, activation="sigmoid" )(out)
 model = Model(inputs=inp, outputs=out)
 model.compile(loss='binary_crossentropy', optimizer=Adam() ,)
 return model
score = cv_rmse(lasso)
 print("lasso: {:.4f}({:.4f})".format(score.mean() , score.std()))
scores['lasso'] =(score.mean() , score.std() )<compute_test_metric>
X_train, X_val, Y_train, Y_val = train_test_split(X, Y, random_state = 2018, test_size = 0.2 )
score = cv_rmse(elasticnet)
 print("elasticnet: {:.4f}({:.4f})".format(score.mean() , score.std()))
scores['elasticnet'] =(score.mean() , score.std() )<compute_test_metric>
val_pred = {}
test_pred = {}
score = cv_rmse(svr)
 print("svr: {:.4f}({:.4f})".format(score.mean() , score.std()))
scores['svr'] =(score.mean() , score.std() )<train_model>
filepath1="weights1_best.h5"
checkpoint = ModelCheckpoint(filepath1, monitor='val_loss', verbose=2, save_best_only=True, mode='min')
 reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.6, patience=1, min_lr=0.0001, verbose=2)
 earlystopping = EarlyStopping(monitor='val_loss', min_delta=0.0001, patience=2, verbose=2, mode='auto')
 callbacks = [checkpoint, reduce_lr]
model = create_capsule(embedding_matrix)
 print(model.summary())
 model.fit(X_train, Y_train, batch_size=512, epochs=6, validation_data=(X_val, Y_val), verbose=2, callbacks=callbacks,)
 model.load_weights(filepath1)
 y_pred = model.predict([X_val], batch_size=1024, verbose=2)
 val_pred['model1'] = y_pred
y_test = model.predict([X_test], batch_size=1024, verbose=2)
 test_pred['model1'] = y_test
f1_score, threshold = f1_smart(np.squeeze(Y_val), np.squeeze(y_pred))
print('Optimal F1: {:.4f} at threshold: {:.4f}'.format(f1_score, threshold))
print('----START Fit----',datetime.now())
 print('Elasticnet')
 elastic_model = elasticnet.fit(X, y_train)
 print('Lasso')
 lasso_model = lasso.fit(X, y_train)
 print('Ridge')
 ridge_model = ridge.fit(X, y_train)
 print('lightgbm')
 lgb_model = lightgbm.fit(X, y_train)
 print('svr')
 svr_model = svr.fit(X, y_train)
 print('stack_gen')
 stack_gen_model = stack_gen.fit(np.array(X), np.array(y_train))<predict_on_test>
clr = CyclicLR(base_lr=0.001, max_lr=0.002,
step_size=300., mode='exp_range',
gamma=0.99994)
 filepath2="weights2_best.h5"
checkpoint = ModelCheckpoint(filepath2, monitor='val_loss', verbose=2, save_best_only=True, mode='min')
 callbacks = [checkpoint, clr]
model = create_lstm_atten(embedding_matrix)
 print(model.summary())
 model.fit(X_train, Y_train, batch_size=1024, epochs=8, validation_data=(X_val, Y_val), verbose=2, callbacks=callbacks,)
 model.load_weights(filepath2)
 y_pred = model.predict([X_val], batch_size=1024, verbose=2)
 val_pred['model2'] = y_pred
y_test = model.predict([X_test], batch_size=1024, verbose=2)
 test_pred['model2'] = y_test
f1_score, threshold = f1_smart(np.squeeze(Y_val), np.squeeze(y_pred))
print('Optimal F1: {:.4f} at threshold: {:.4f}'.format(f1_score, threshold))
def blend_predictions(X):
return(( 0.12 * elastic_model.predict(X)) + \ 
(0.12 * lasso_model.predict(X)) + \ 
(0.12 * ridge_model.predict(X)) + \ 
(0.22 * lgb_model.predict(X)) + \ 
(0.1 * svr_model.predict(X)) + \ 
(0.32 * stack_gen_model.predict(np.array(X))))<compute_test_metric>
filepath3="weights3_best.h5"
checkpoint = ModelCheckpoint(filepath3, monitor='val_loss', verbose=2, save_best_only=True, mode='min')
 reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.6, patience=1, min_lr=0.0001, verbose=2)
 earlystopping = EarlyStopping(monitor='val_loss', min_delta=0.0001, patience=2, verbose=2, mode='auto')
 callbacks = [checkpoint, reduce_lr]
model = create_cnn(embedding_matrix)
 print(model.summary())
 model.fit(X_train, Y_train, batch_size=512, epochs=10, validation_data=(X_val, Y_val), verbose=2, callbacks=callbacks,)
 model.load_weights(filepath3)
 y_pred = model.predict([X_val], batch_size=1024, verbose=2)
 val_pred['model3'] = y_pred
y_test = model.predict([X_test], batch_size=1024, verbose=2)
 test_pred['model3'] = y_test
f1_score, threshold = f1_smart(np.squeeze(Y_val), np.squeeze(y_pred))
print('Optimal F1: {:.4f} at threshold: {:.4f}'.format(f1_score, threshold))
blended_score = rmsle(y_train, blend_predictions(X))
scores['blended'] =(blended_score, 0)
 print('RMSLE score on train data:')
 print(blended_score )<load_from_csv>
filepath4="weights4_best.h5"
checkpoint = ModelCheckpoint(filepath4, monitor='val_loss', verbose=2, save_best_only=True, mode='min')
 reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.6, patience=1, min_lr=0.0001, verbose=2)
 earlystopping = EarlyStopping(monitor='val_loss', min_delta=0.0001, patience=2, verbose=2, mode='auto')
 callbacks = [checkpoint, reduce_lr]
model = create_capsule(embedding_matrix_1)
 print(model.summary())
 model.fit(X_train, Y_train, batch_size=512, epochs=6, validation_data=(X_val, Y_val), verbose=2, callbacks=callbacks,)
 model.load_weights(filepath4)
 y_pred = model.predict([X_val], batch_size=1024, verbose=2)
 val_pred['model4'] = y_pred
y_test = model.predict([X_test], batch_size=1024, verbose=2)
 test_pred['model4'] = y_test
f1_score, threshold = f1_smart(np.squeeze(Y_val), np.squeeze(y_pred))
print('Optimal F1: {:.4f} at threshold: {:.4f}'.format(f1_score, threshold))
submission = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/sample_submission.csv")
 submission.iloc[:,1] = np.floor(np.expm1(blend_predictions(X_test)) )<save_to_csv>
from sklearn.metrics import f1_score as f1
submission.to_csv("submission.csv", index=False )<save_to_csv>
stack_x = np.concatenate([i for i in val_pred.values() ], axis = 1)
 stack_test = np.concatenate([i for i in test_pred.values() ], axis = 1 )
submission.to_csv("submission.csv", index=False )<set_options>
y_test = np.zeros(( 56370,))
%matplotlib inline
warnings.filterwarnings("ignore")
 print('Versions:')
 print(' python', platform.python_version())
 n =('numpy', 'pandas', 'sklearn', 'matplotlib', 'seaborn')
 nn =(np, pd, sklearn, mpl, sns)
 for a, b in zip(n, nn):
print(' --', str(a), b.__version__ )<set_options>
stack_train, oof_train, stack_train_target, oof_target = train_test_split(stack_x, Y_val, random_state = 2018, test_size = 0.2 )
pd.set_option('colheader_justify', 'left')
 pd.set_option('precision', 0)
 pd.options.display.float_format = '{:,.2f}'.format
pd.set_option('display.max_colwidth', -1 )<set_options>
oof_test = np.zeros(oof_train.shape[0] )
sns.set_style('whitegrid', { 'axes.axisbelow': True, 'axes.edgecolor': 'black', 'axes.facecolor': 'white',
'axes.grid': True, 'axes.labelcolor': 'black', 'axes.spines.bottom': True, 'axes.spines.left': True,
'axes.spines.right': False, 'axes.spines.top': False, 'figure.facecolor': 'white',
'grid.color': 'grey', 'grid.linestyle': ':', 'image.cmap': 'rocket', 'lines.solid_capstyle': 'round',
'patch.edgecolor': 'w', 'patch.force_edgecolor': True, 'text.color': 'black',
'xtick.top': False, 'xtick.bottom': True, 'xtick.color': 'navy', 'xtick.direction': 'out',
'ytick.right': False, 'ytick.left': True, 'ytick.color': 'navy', 'ytick.direction': 'out'} )<import_modules>
params = {}
params['learning_rate'] = 0.05
params['boosting_type'] = 'gbdt'
params['objective'] = 'binary'
params['metric'] = 'binary_logloss'
params['sub_feature'] = 0.5
params['num_leaves'] = 10
params['min_data'] = 50
params['max_depth'] = 6
from sklearn import svm, tree, linear_model, neighbors, naive_bayes, ensemble, discriminant_analysis, gaussian_process
from sklearn import feature_selection, model_selection, metrics
from sklearn.preprocessing import OneHotEncoder, LabelEncoder
from xgboost import XGBClassifier<load_from_csv>
kfold = StratifiedKFold(n_splits=10, random_state=2018, shuffle=True )
train_raw = pd.read_csv('.. /input/titanic/train.csv')
 test_raw = pd.read_csv('.. /input/titanic/test.csv')
 len(train_raw)+ len(test_raw )<concatenate>
for i,(train_index, valid_index)in enumerate(kfold.split(stack_train, stack_train_target)) :
stack_x_train, stack_x_val, stack_y_train, stack_y_val = stack_train[train_index], stack_train[valid_index], stack_train_target[train_index], stack_train_target[valid_index]
d_train = lgb.Dataset(stack_x_train, label=stack_y_train)
 clf = lgb.train(params, d_train, 100)
 stack_y_pred =(clf.predict(stack_x_val)> 0.36 ).astype(int)
 f1_score = f1(stack_y_val, stack_y_pred)
 print('f1 score for fold ', i, ': ', f1_score)
 y_test +=(clf.predict(stack_test)) /10
oof_test +=(clf.predict(oof_train)) /10
df = pd.concat(objs=[train_raw, test_raw], axis=0)
 df.shape<load_from_csv>
for i in [0.30, 0.31, 0.32, 0.33, 0.34, 0.35, 0.36, 0.37, 0.38]:
print('threshold: ', i, "f1: ", f1(oof_target,(oof_test > i ).astype(int)) )
<drop_column>
y_test = y_test.reshape(( -1, 1))
pred_test_y =(y_test>0.36 ).astype(int)
 sub['prediction'] = pred_test_y
sub.to_csv("submission.csv", index=False )
col1 = test_raw['PassengerId']
df.drop(['PassengerId', 'Cabin'], axis=1, inplace=True )<feature_engineering>
def tune_seed(seed):
stack_train, oof_train, stack_train_target, oof_target = train_test_split(stack_x, Y_val, random_state = seed, test_size = 0.2)
 oof_test = np.zeros(oof_target.size)
 for i,(train_index, valid_index)in enumerate(kfold.split(stack_train, stack_train_target)) :
stack_x_train, stack_x_val, stack_y_train, stack_y_val = stack_train[train_index], stack_train[valid_index], stack_train_target[train_index], stack_train_target[valid_index]
d_train = lgb.Dataset(stack_x_train, label=stack_y_train)
 clf = lgb.train(params, d_train, 100)
 oof_test +=(clf.predict(oof_train)) /10
score = f1(oof_target,(oof_test > 0.34 ).astype(int))
return score
df['Age'] = df['Age'].fillna(df['Age'].median())
 df['Fare'] = df['Fare'].fillna(df['Fare'].mean())
 df['Embarked'] = df['Embarked'].fillna('S')
 df.isnull().sum().to_frame().T<feature_engineering>
holdout_scores = []
for s in range(1999, 2020):
holdout_score = tune_seed(s)
 print('f1 score at seed ', s, ' is ', holdout_score )
df['Fsize'] = df['Parch'] + df['SibSp'] + 1<feature_engineering>
np.sum(oof_target)/(len(oof_target))
df['Surname'], df['Name'] = zip(*df['Name'].apply(lambda x: x.split(',')))
 df['Title'], df['Name'] = zip(*df['Name'].apply(lambda x: x.split('.')))
 titles =(df['Title'].value_counts() < 10)
 df['Title'] = df['Title'].apply(lambda x: ' Misc' if titles.loc[x] == True else x)
 df['Title'].value_counts().to_frame().T<count_unique_values>
np.sum(pred_test_y)/len(pred_test_y )