Kaggle competition: Quora Insincere Questions Classification
<SOS> metric: FScore_1 Kaggle data source: quora-insincere-questions-classification<feature_engineering>
def f1() :
return
idx = int(data_train.shape[0])
 data_train['Age'] = data.iloc[:idx].Age
data_test['Age'] = data.iloc[idx:].Age<categorify>
le = LabelEncoder()
data_train_LE = data_train.copy()
data_test_LE = data_test.copy()
columns = ['Sex', 'Embarked', 'ticket_type', 'cabin_type', 'title']
for col in columns:
le.fit(data_train_LE[col])
 data_train_LE[col] = le.fit_transform(data_train_LE[col])
 data_test_LE[col] = le.transform(data_test_LE[col])
 data_train_LE.head()<drop_column>
def set_all_parameters() :
global epoc_used
global mn
global batch_size
global patience
global nsplits
global stop_split
global num_lstm
global num_dense
global rate_drop_lstm
global rate_drop_dense
global rate_drop_spatial
global loss
global act
global opt
global met
global es_mon
global es_mode
global Trainable
global lr
global method
global model_list
global maxlen
global max_features
global pretext_proc
global bstart
sm ="6"
ssh = 50
ssd = 50
sdh = 0.1
sdd = 0.1
sds =0.1
loss_bin='binary_crossentropy'
opt_adam="adam"
opt_rmsprop="rmsprop"
met_acc=["accuracy"]
met_f1=[f1]
mon_acc = "val_acc"
mode_acc = "max"
mon_f1 = "val_f1"
mode_f1 = "max"
mon_loss ="val_loss"
mode_loss="min"
ns=4
ss=4
bs=1024
ne=8
pl=3
bstart = time.time()
model_list=[]
model_list.append(["18",1,"6","Concatenated",10,1,1024,4,4,50,50,0.1,0.1,0.15,'binary_crossentropy','rmsprop',["accuracy"],'val_acc','max'])
 maxlen = 75
max_features = None
pretext_proc=True
Trainable=False
drop_col = ['Age_mean', 'SibSp', 'Parch']
data_train_LE = data_train_LE.drop(drop_col, axis=1)
 data_test_LE = data_test_LE.drop(drop_col, axis=1 )<prepare_x_and_y>
set_all_parameters()
X_train_onehot = data_train.drop(drop_col, axis=1)
 X_test_onehot = data_test.drop(drop_col, axis=1 )<categorify>
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):

 self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) , K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim
columns = ['cabin_type', 'title', 'Sex', 'Embarked', 'ticket_type', 'Pclass']
for col in columns:
X_train_onehot = pd.concat([X_train_onehot, pd.get_dummies(X_train_onehot[col], drop_first = True)], axis =1)
 X_test_onehot = pd.concat([X_test_onehot, pd.get_dummies(X_test_onehot[col], drop_first = True)], axis =1)
<categorify>
class AttentionWeightedAverage(Layer):

 def __init__(self, return_attention=False, **kwargs):
self.init = initializers.get('uniform')
 self.supports_masking = True
self.return_attention = return_attention
super(AttentionWeightedAverage, self ).__init__(** kwargs)
 def build(self, input_shape):
self.input_spec = [InputSpec(ndim=3)]
assert len(input_shape)== 3
self.W = self.add_weight(shape=(input_shape[2], 1),
name='{}_W'.format(self.name),
initializer=self.init)
 self.trainable_weights = [self.W]
super(AttentionWeightedAverage, self ).build(input_shape)
 def call(self, x, mask=None):
logits = K.dot(x, self.W)
 x_shape = K.shape(x)
 logits = K.reshape(logits,(x_shape[0], x_shape[1]))
ai = K.exp(logits - K.max(logits, axis=-1, keepdims=True))
if mask is not None:
mask = K.cast(mask, K.floatx())
 ai = ai * mask
att_weights = ai /(K.sum(ai, axis=1, keepdims=True)+ K.epsilon())
 weighted_input = x * K.expand_dims(att_weights)
 result = K.sum(weighted_input, axis=1)
 if self.return_attention:
return [result, att_weights]
return result
def get_output_shape_for(self, input_shape):
return self.compute_output_shape(input_shape)
 def compute_output_shape(self, input_shape):
output_len = input_shape[2]
if self.return_attention:
return [(input_shape[0], output_len),(input_shape[0], input_shape[1])]
return(input_shape[0], output_len)
 def compute_mask(self, input, input_mask=None):
if isinstance(input_mask, list):
return [None] * len(input_mask)
 else:
return None
X_train_onehot = X_train_onehot.drop(columns, axis=1)
 X_test_onehot = X_test_onehot.drop(columns, axis=1 )<prepare_x_and_y>
class CyclicLR(Callback):

 def __init__(self, base_lr=0.001, max_lr=0.006, step_size=2000., mode='triangular',
gamma=1., scale_fn=None, scale_mode='cycle'):
super(CyclicLR, self ).__init__()
self.base_lr = base_lr
self.max_lr = max_lr
self.step_size = step_size
self.mode = mode
self.gamma = gamma
if scale_fn == None:
if self.mode == 'triangular':
self.scale_fn = lambda x: 1.
 self.scale_mode = 'cycle'
elif self.mode == 'triangular2':
self.scale_fn = lambda x: 1/(2.**(x-1))
self.scale_mode = 'cycle'
elif self.mode == 'exp_range':
self.scale_fn = lambda x: gamma**(x)
 self.scale_mode = 'iterations'
else:
self.scale_fn = scale_fn
self.scale_mode = scale_mode
self.clr_iterations = 0.
 self.trn_iterations = 0.
 self.history = {}
self._reset()
def _reset(self, new_base_lr=None, new_max_lr=None,
new_step_size=None):

 if new_base_lr != None:
self.base_lr = new_base_lr
if new_max_lr != None:
self.max_lr = new_max_lr
if new_step_size != None:
self.step_size = new_step_size
self.clr_iterations = 0.
 def clr(self):
cycle = np.floor(1+self.clr_iterations/(2*self.step_size))
x = np.abs(self.clr_iterations/self.step_size - 2*cycle + 1)
 if self.scale_mode == 'cycle':
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(cycle)
 else:
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(self.clr_iterations)
 def on_train_begin(self, logs={}):
logs = logs or {}
if self.clr_iterations == 0:
K.set_value(self.model.optimizer.lr, self.base_lr)
 else:
K.set_value(self.model.optimizer.lr, self.clr())
 def on_batch_end(self, epoch, logs=None):
logs = logs or {}
self.trn_iterations += 1
self.clr_iterations += 1
self.history.setdefault('lr', [] ).append(K.get_value(self.model.optimizer.lr))
self.history.setdefault('iterations', [] ).append(self.trn_iterations)
 for k, v in logs.items() :
self.history.setdefault(k, [] ).append(v)
 K.set_value(self.model.optimizer.lr, self.clr())
X_train_lab = data_train.drop(drop_col, axis=1)
 X_test_lab = data_test.drop(drop_col, axis=1 )<categorify>
def check_coverage(vocab,embeddings_index):
try:
print("Len - embeddings_index:",len(embeddings_index))
except :
print("Len - embeddings_index:",len(embeddings_index.index2word))
print("Len -",len(vocab))
a = {}
oov = {}
k = 0
i = 0
for word in tqdm(vocab):
try:
a[word] = embeddings_index[word]
k += vocab[word]
except:
oov[word] = vocab[word]
i += vocab[word]
pass
print("Words Found:",k)
 print("Words Not Found:",i)
 print("Total Words:",i+k)
 print('Found embeddings for {:.2%} of vocab'.format(len(a)/ len(vocab)))
 print('Found embeddings for {:.2%} of all text'.format(k /(k + i)))
 sorted_x = sorted(oov.items() , key=operator.itemgetter(1)) [::-1]
return sorted_x
le = LabelEncoder()
columns = ['Sex', 'Embarked', 'ticket_type', 'cabin_type', 'title']
for col in columns:
le.fit(data_train[col])
 X_train_lab[col] = le.transform(X_train_lab[col])
 X_test_lab[col] = le.transform(X_test_lab[col])
 X_test_lab.head()<drop_column>
def squash(x, axis=-1):
s_squared_norm = K.sum(K.square(x), axis, keepdims=True)
 scale = K.sqrt(s_squared_norm + K.epsilon())
 return x / scale
class Capsule(Layer):
def __init__(self, num_capsule, dim_capsule, routings=3, kernel_size=(9, 1), share_weights=True,
activation='default', **kwargs):
super(Capsule, self ).__init__(**kwargs)
 self.num_capsule = num_capsule
self.dim_capsule = dim_capsule
self.routings = routings
self.kernel_size = kernel_size
self.share_weights = share_weights
if activation == 'default':
self.activation = squash
else:
self.activation = Activation(activation)
 def build(self, input_shape):
super(Capsule, self ).build(input_shape)
 input_dim_capsule = input_shape[-1]
if self.share_weights:
self.W = self.add_weight(name='capsule_kernel',
shape=(1, input_dim_capsule,
self.num_capsule * self.dim_capsule),
initializer='glorot_uniform',
trainable=True)
 else:
input_num_capsule = input_shape[-2]
self.W = self.add_weight(name='capsule_kernel',
shape=(input_num_capsule,
input_dim_capsule,
self.num_capsule * self.dim_capsule),
initializer='glorot_uniform',
trainable=True)
 def call(self, u_vecs):
if self.share_weights:
u_hat_vecs = K.conv1d(u_vecs, self.W)
 else:
u_hat_vecs = K.local_conv1d(u_vecs, self.W, [1], [1])
 batch_size = K.shape(u_vecs)[0]
input_num_capsule = K.shape(u_vecs)[1]
u_hat_vecs = K.reshape(u_hat_vecs,(batch_size, input_num_capsule,
self.num_capsule, self.dim_capsule))
u_hat_vecs = K.permute_dimensions(u_hat_vecs,(0, 2, 1, 3))
b = K.zeros_like(u_hat_vecs[:, :, :, 0])
 for i in range(self.routings):
b = K.permute_dimensions(b,(0, 2, 1))
c = K.softmax(b)
 c = K.permute_dimensions(c,(0, 2, 1))
b = K.permute_dimensions(b,(0, 2, 1))
outputs = self.activation(K.batch_dot(c, u_hat_vecs, [2, 2]))
if i < self.routings - 1:
b = K.batch_dot(outputs, u_hat_vecs, [2, 3])
 return outputs
def compute_output_shape(self, input_shape):
return(None, self.num_capsule, self.dim_capsule )
X_train_mean = data_train.drop(drop_col, axis=1)
 X_test_mean = data_test.drop(drop_col, axis=1 )<categorify>
def remove_urls(vTEXT):
vTEXT = re.sub(r'(https|http)?:\/\/(\w|\.|\/|\?|\=|\&|\%)*\b', '', vTEXT, flags=re.MULTILINE)
 return(vTEXT)
 def ReplaceThreeOrMore(s):
pattern = re.compile(r" (.) \1{2,}", re.DOTALL)
 return pattern.sub(r"\1", s)
 def splitstring(s):
proposed_pattern = s[0]
for i, c in enumerate(s[1:], 1):
if c != " ":
if proposed_pattern == s[i:(i+len(proposed_pattern)) ]:
break
else:
proposed_pattern += c
else:
exit(1)
 return proposed_pattern
def clean_numbers(x):
x = re.sub('[0-9]{5,}', '
x = re.sub('[0-9]{4}', '
x = re.sub('[0-9]{3}', '
x = re.sub('[0-9]{2}', '
return x
special_character_removal=re.compile(r'[^a-z\d ]',re.IGNORECASE)
 replace_numbers=re.compile(r'\d+',re.IGNORECASE)
 def _get_mispell(mispell_dict):
mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))
return mispell_dict, mispell_re
mispell_dict = {'colour':'color',
'centre':'center',
'didnt':'did not',
'doesnt':'does not',
'isnt':'is not',
'shouldnt':'should not',
'favourite':'favorite',
'travelling':'traveling',
'counselling':'counseling',
'theatre':'theater',
'cancelled':'canceled',
'labour':'labor',
'organisation':'organization',
'wwii':'world war 2',
'citicise':'criticize',
'instagram': 'social medium',
'whatsapp': 'social medium',
'snapchat': 'social medium'
}
mispellings, mispellings_re = _get_mispell(mispell_dict)
 def replace_typical_misspell(text):
def replace(match):
return mispellings[match.group(0)]
return mispellings_re.sub(replace, text)
 def clean_text(x):
x = str(x)
 for punct in "/-'":
x = x.replace(punct, ' ')
 for punct in '&':
x = x.replace(punct, f' {punct} ')
 for punct in '?!.,"
x = x.replace(punct, '')
 return x
puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…',
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─',
'▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]
def clean_text_NOTUSED(x):
x = str(x)
 for punct in puncts:
x = x.replace(punct, f' {punct} ')
 return x
def text_to_wordlist(text,to_lower=True, rem_urls=False, rem_3plus=False,
clean_t=True, clean_num=True,mispelling=True,rem_specwords= False,
split_repeated=True, rem_special=False, rep_num=False,
man_adj=True, rem_stopwords=False, stem_snowball=False,
stem_porter=False, lemmatize=False):
if rem_urls:
text = remove_urls(text)
 if to_lower:
text = text.lower()
if rem_3plus:
text = ReplaceThreeOrMore(text)
 if clean_t:
text= clean_text(text)
 if clean_num:
text= clean_numbers(text)
 if mispelling:
text= replace_typical_misspell(text)
 if man_adj:
text = re.sub(r"[^A-Za-z0-9^,!.\/'+-=]", " ", text)
 text = re.sub(r"what's", "what is ", text)
 text = re.sub(r"'s", " ", text)
 text = re.sub(r"'ve", " have ", text)
 text = re.sub(r"can't", "cannot ", text)
 text = re.sub(r"n't", " not ", text)
 text = re.sub(r"i'm", "i am ", text)
 text = re.sub(r"'re", " are ", text)
 text = re.sub(r"'d", " would ", text)
 text = re.sub(r"'ll", " will ", text)
 text = re.sub(r",", " ", text)
 text = re.sub(r"\.", " ", text)
 text = re.sub(r"!", " ! ", text)
 text = re.sub(r"\/", " ", text)
 text = re.sub(r"\^", " ^ ", text)
 text = re.sub(r"\+", " + ", text)
 text = re.sub(r"\-", " - ", text)
 text = re.sub(r"\=", " = ", text)
 text = re.sub(r"'", " ", text)
 text = re.sub(r"(\d+ )(k)", r"\g<1>000", text)
 text = re.sub(r":", " : ", text)
 text = re.sub(r" e g ", " eg ", text)
 text = re.sub(r" b g ", " bg ", text)
 text = re.sub(r" u s ", " american ", text)
 text = re.sub(r"\0s", "0", text)
 text = re.sub(r" 9 11 ", "911", text)
 text = re.sub(r"e - mail", "email", text)
 text = re.sub(r"j k", "jk", text)
 text = re.sub(r"\s{2,}", " ", text)
 text = text.split()
if split_repeated:
for i, c in enumerate(text):
text[i]=splitstring(c)
 if rem_specwords:
to_remove = ['a','to','of','and']
text = [w for w in text if not w in to_remove]
if rem_stopwords:
stops = set(stopwords.words("english"))
text = [w for w in text if not w in stops]
text = " ".join(text)
 if rem_special:
text=special_character_removal.sub('',text)
 if rep_num:
text=replace_numbers.sub('n',text)
 if stem_snowball:
text = text.split()
stemmer = SnowballStemmer('english')
 stemmed_words = [stemmer.stem(word)for word in text]
text = " ".join(stemmed_words)
 if stem_porter:
st = PorterStemmer()
txt = " ".join([st.stem(w)for w in text.split() ])
 if lemmatize:
wordnet_lemmatizer = WordNetLemmatizer()
txt = " ".join([wordnet_lemmatizer.lemmatize(w)for w in text.split() ])
 return(text )
columns = ['cabin_type', 'title', 'Sex', 'Embarked', 'ticket_type']
for col in columns:
ordered_labels = X_train_mean.groupby([col])['Survived'].mean().to_dict()
X_train_mean[col] = X_train_mean[col].map(ordered_labels)
 X_test_mean[col] = X_test_mean[col].map(ordered_labels )<drop_column>
def save_tokenizer(file, tokenizer):
with open(file, 'wb')as handle:
pickle.dump(tokenizer, handle, protocol=pickle.HIGHEST_PROTOCOL)
 def load_tokenizer(file, tokenizer):
with open(file, 'rb')as handle:
tokenizer = pickle.load(handle)
 return tokenizer
X_train_freq = data_train.drop(drop_col, axis=1)
 X_test_freq = data_test.drop(drop_col, axis=1 )<categorify>
def load_and_prec(PreProcess=False):
global max_features
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 print("Train shape : ",train_df.shape)
 print("Test shape : ",test_df.shape)
 train_X=train_df["question_text"].values
test_X=test_df["question_text"].values
tokenizer = Tokenizer(num_words=max_features)
 if PreProcess:
train_questions = []
for text in train_X:
train_questions.append(text_to_wordlist(text))
test_questions=[]
for text in test_X:
test_questions.append(text_to_wordlist(text))
tokenizer.fit_on_texts(train_questions+test_questions)
 train_X = tokenizer.texts_to_sequences(train_questions)
 test_X = tokenizer.texts_to_sequences(test_questions)
 else:
tokenizer.fit_on_texts(list(train_X)+list(test_X))
train_X = tokenizer.texts_to_sequences(train_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 print(len(train_X), 'train sequences')
 print(len(test_X), 'test sequences')
 print('Average train sequence length: {}'.format(np.mean(list(map(len, train_X)) , dtype=int)))
 print('Average test sequence length: {}'.format(np.mean(list(map(len, test_X)) , dtype=int)))
 print('Max train sequence length: {}'.format(np.max(list(map(len, train_X)))))
 print('Max test sequence length: {}'.format(np.max(list(map(len, test_X)))))
 train_X = pad_sequences(train_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 train_y = train_df['target'].values
print(len(tokenizer.word_index))
print(len(tokenizer.word_counts))
if(max_features==None):
max_features = len(tokenizer.word_index)+1
return train_X, test_X, train_y, test_df, tokenizer.word_index
columns = ['cabin_type', 'title', 'Sex', 'Embarked', 'ticket_type']
for col in columns:
ordered_labels = X_train_freq[col].value_counts().to_dict()
X_train_freq[col] = X_train_freq[col].map(ordered_labels)
 X_test_freq[col] = X_test_freq[col].map(ordered_labels )<define_variables>
def create_model0(embedding_matrix):
mdln="0-Bidirec(CuDNNGRU)-GlobalMax-Dense-Dropout"
input_layer = Input(shape=(maxlen,))
embedding_layer = Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],
weights=[embedding_matrix], trainable=Trainable )(input_layer)
 x = LSTM(num_lstm, return_sequences=True )(embedding_layer)
 x = GlobalMaxPool1D()(x)
 x = Dense(num_dense, activation="relu" )(x)
 x = Dropout(rate_drop_dense )(x)
 preds = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=input_layer, outputs=preds)
 model.compile(loss='binary_crossentropy', optimizer=opt, metrics=met)
 return model, mdln
random_state = 4<choose_model_class>
def create_model1(embedding_matrix):
mdln="1-LSTM-Dropout-Attention-Dense-Dropout-BatchNormalization"
input_layer = Input(shape=(maxlen,))
embedding_layer = Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],
weights=[embedding_matrix], trainable=Trainable )(input_layer)
 x = LSTM(num_lstm, dropout=rate_drop_lstm, recurrent_dropout=rate_drop_lstm,return_sequences=True )(embedding_layer)
 x = Dropout(rate_drop_lstm )(x)
 x = Attention(maxlen )(x)
 x = Dense(num_dense, activation="relu" )(x)
 x = Dropout(rate_drop_dense )(x)
 x = BatchNormalization()(x)
 preds = Dense(1, activation='sigmoid' )(x)
 model = Model(inputs=input_layer, outputs=preds)
 model.compile(loss='binary_crossentropy',optimizer=opt, metrics=met)
 return model, mdln
kfold = StratifiedKFold(n_splits=5 )<split>
def create_model2(embedding_matrix):
mdln="2-Bidirect(CuDNNLSTM)-GlobalMax-Dense-Dropout-Dense-Dropout"
input_layer = Input(shape=(maxlen,))
embedding_layer = Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],
weights=[embedding_matrix], trainable=Trainable )(input_layer)
 x = Bidirectional(CuDNNLSTM(num_lstm, return_sequences=True))(embedding_layer)
 x = GlobalMaxPool1D()(x)
 x = Dense(num_dense, activation="relu" )(x)
 x = Dropout(rate_drop_dense )(x)
 x = Dense(num_dense, activation="relu" )(x)
 x = Dropout(rate_drop_dense )(x)
 preds = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=input_layer, outputs=preds)
 model.compile(loss='binary_crossentropy',optimizer=opt, metrics=met)
 return model, mdln
def separate(X_train):
X = X_train.drop(columns= ['Survived'])
 Y = X_train['Survived']
return X, Y<split>
def create_model3(embedding_matrix):
mdln="3-Bidirect(CuDNNLSTM)-Bidirect(CuDNNLSTM)-Attention-Dense"
input_layer = Input(shape=(maxlen,))
embedding_layer = Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],
weights=[embedding_matrix], trainable=Trainable )(input_layer)
 num_lstm2=int(num_lstm/2)
 x = Bidirectional(CuDNNLSTM(num_lstm, return_sequences=True))(embedding_layer)
 x = Bidirectional(CuDNNLSTM(num_lstm2, return_sequences=True))(x)
 x = Attention(maxlen )(x)
 x = Dense(num_dense, activation="relu" )(x)
 preds = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=input_layer, outputs=preds)
 model.compile(loss='binary_crossentropy', optimizer=Adam(lr=1e-3),metrics=met)
 return model, mdln
X_onehot, Y_onehot = separate(X_train_onehot)
 X_lab, Y_lab = separate(X_train_lab)
 X_mean, Y_mean = separate(X_train_mean)
 X_freq, Y_freq = separate(X_train_freq )<choose_model_class>
def create_model4(embedding_matrix):
mdln="4-Reshape-Concat(Conv2D(Filters)+MaxPool2d)-Flatten-Dropout"
filter_sizes = [1,2,3,5]
num_filters = 36
input_layer = Input(shape=(maxlen,))
embedding_layer = Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],
weights=[embedding_matrix], trainable=Trainable )(input_layer)
 x = Reshape(( maxlen, embedding_matrix.shape[1], 1))(embedding_layer)
 maxpool_pool = []
for i in range(len(filter_sizes)) :
conv = Conv2D(num_filters, kernel_size=(filter_sizes[i], embedding_matrix.shape[1]),
kernel_initializer='he_normal', activation='elu' )(x)
 maxpool_pool.append(MaxPool2D(pool_size=(maxlen - filter_sizes[i] + 1, 1))(conv))
z = Concatenate(axis=1 )(maxpool_pool)
 z = Flatten()(z)
 z = Dropout(0.1 )(z)
 preds = Dense(1, activation="sigmoid" )(z)
 model = Model(inputs=input_layer, outputs=preds)
 model.compile(loss='binary_crossentropy',optimizer=opt, metrics=met)
 return model, mdln
random_state = 4
classifiers = []
classifiers.append(( 'SVC', make_pipeline(StandardScaler() ,SVC(random_state=random_state))))
classifiers.append(( 'DecisionTree', DecisionTreeClassifier(random_state=random_state)))
 classifiers.append(( 'AdaBoost', AdaBoostClassifier(DecisionTreeClassifier(random_state=random_state),\
random_state=random_state,learning_rate=0.1)))
 classifiers.append(( 'RandomForest', RandomForestClassifier(random_state=random_state)))
 classifiers.append(( 'GradientBoost', GradientBoostingClassifier(random_state=random_state)))
 classifiers.append(( 'MPL', make_pipeline(StandardScaler() , MLPClassifier(random_state=random_state))))
classifiers.append(( 'KNN',make_pipeline(MinMaxScaler() ,KNeighborsClassifier(n_neighbors=7))))
results = []
names = []
for name, classifier in classifiers:
kfold = model_selection.KFold(n_splits= 3, random_state=random_state, shuffle = True)
 cv_results = model_selection.cross_val_score(classifier, X_onehot, y = Y_onehot, cv=kfold, scoring='accuracy')
 results.append(cv_results)
 names.append(name)
 msg = "%s: %f(%f)" %(name, cv_results.mean() , cv_results.std())
 print(msg)
<train_on_grid>
def create_model5(embedding_matrix):
mdln="05-conv-max-conv-max-bulstm-max-dd"
input_layer = Input(shape=(maxlen,))
embedding_layer = Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],
weights=[embedding_matrix], trainable=Trainable )(input_layer)
 x = Dropout(0.2 )(embedding_layer)
 x = Conv1D(filters=embedding_matrix.shape[1], kernel_size=4, padding='same', activation='relu' )(x)
 x = MaxPooling1D(pool_size=2 )(x)
 x = Conv1D(filters=embedding_matrix.shape[1], kernel_size=4, padding='same', activation='relu' )(x)
 x = MaxPooling1D(pool_size=2 )(x)
 x = Bidirectional(CuDNNLSTM(num_lstm, return_sequences=True))(x)
 x = GlobalMaxPool1D()(x)
 x = Dense(num_dense, activation="relu" )(x)
 x = Dropout(rate_drop_dense )(x)
 x = Dense(num_dense, activation="relu" )(x)
 x = Dropout(rate_drop_dense )(x)
 preds = Dense(1, activation='sigmoid' )(x)
 model = Model(inputs=input_layer, outputs=preds)
 model.compile(loss='binary_crossentropy',optimizer=opt, metrics=met)
 return model, mdln
def random_forest(X, Y, X_test):
parameters = {'max_depth' : [2, 4, 5, 10],
'n_estimators' : [200, 500, 1000, 2000],
'min_samples_split' : [3, 4, 5],
}
kfold = model_selection.KFold(n_splits=3, random_state=random_state, shuffle = True)
 model_RFC = RandomForestClassifier(random_state = 4, n_jobs = -1)
 search_RFC = GridSearchCV(model_RFC, parameters, n_jobs = -1, cv = kfold, scoring = 'accuracy',verbose=1)
 search_RFC.fit(X, Y)
 predicted= search_RFC.predict(X_test)
 print("Best parameters are:", search_RFC.best_params_)
 print("Best accuracy achieved:",search_RFC.best_score_)
 return search_RFC.best_params_, model_RFC, search_RFC, predicted<split>
def create_model6(embedding_matrix):
mdln="6-Bidirect(CuDNNGRU)-Cocat(GlobalMax+Global Avarage)- Dense - Dropout"
input_layer = Input(shape=(maxlen,))
embedding_layer = Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],
weights=[embedding_matrix], trainable=Trainable )(input_layer)
 x = Bidirectional(CuDNNGRU(num_lstm, return_sequences=True))(embedding_layer)
 tower_1 = GlobalMaxPool1D()(x)
 tower_2 = GlobalAveragePooling1D()(x)
 output = concatenate([ tower_1, tower_2])
 x = Dense(num_dense, activation="relu" )(output)
 x = Dropout(rate_drop_dense )(x)
 preds = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=input_layer, outputs=preds)
 model.compile(loss='binary_crossentropy',optimizer=opt,metrics=met)
 return model, mdln
param_RFC_onehot, model_RFC_onehot, search_RFC_onehot, predicted_cv_RFC_onehot = random_forest(X_onehot, Y_onehot, X_test_onehot)
 param_RFC_lab, model_RFC_lab, search_RFC_lab, predicted_cv_RFC_lab = random_forest(X_lab, Y_lab, X_test_lab)
 param_RFC_mean, model_RFC_mean, search_RFC_mean, predicted_cv_RFC_mean = random_forest(X_mean, Y_mean, X_test_mean)
 param_RFC_freq, model_RFC_freq, search_RFC_freq, predicted_cv_RFC_freq = random_forest(X_freq, Y_freq, X_test_freq )<train_model>
def create_model7(embedding_matrix):
mdln="7-Bidirect(CuDNNGRU)-Attention-Dense-Dropout"
input_layer = Input(shape=(maxlen,))
embedding_layer = Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],
weights=[embedding_matrix], trainable=Trainable )(input_layer)
 x = Bidirectional(CuDNNGRU(num_lstm, return_sequences=True))(embedding_layer)
 x = Attention(maxlen )(x)
 x = Dense(num_dense, activation="relu" )(x)
 x = Dropout(rate_drop_dense )(x)
 preds = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=input_layer, outputs=preds)
 model.compile(loss='binary_crossentropy', optimizer=opt, metrics=met)
 return model , mdln
def fit_pred_RF(X, Y, X_test):
model_RFC = RandomForestClassifier(max_depth =2, min_samples_split =3, n_estimators = 5000,
random_state = 4, n_jobs = -1)
 model_RFC.fit(X, Y)
 predicted= model_RFC.predict(X_test)
 return predicted, model_RFC 
<compute_train_metric>
def create_model8(embedding_matrix):
mdln="8-Bidirect(CuDNNGRU)-Concat(GlobalMax+GlobalAverage)-Dense-Dropout"
input_layer = Input(shape=(maxlen,))
embedding_layer = Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],
weights=[embedding_matrix], trainable=Trainable )(input_layer)
 x = Bidirectional(CuDNNGRU(num_lstm, return_sequences=True))(embedding_layer)
 avg_pool = GlobalAveragePooling1D()(x)
 max_pool = GlobalMaxPooling1D()(x)
 conc = concatenate([avg_pool, max_pool])
 conc = Dense(num_dense, activation="relu" )(conc)
 conc = Dropout(rate_drop_dense )(conc)
 preds = Dense(1, activation="sigmoid" )(conc)
 model = Model(inputs=input_layer, outputs=preds)
 model.compile(loss='binary_crossentropy', optimizer=opt, metrics=met)
 return model, mdln
predicted_RFC_onehot, model_RFC_onehot = fit_pred_RF(X_onehot, Y_onehot, X_test_onehot)
 predicted_RFC_lab, model_RFC_lab = fit_pred_RF(X_lab, Y_lab, X_test_lab)
 predicted_RFC_mean, model_RFC_mean = fit_pred_RF(X_mean, Y_mean, X_test_mean)
 predicted_RFC_freq, model_RFC_freq = fit_pred_RF(X_freq, Y_freq, X_test_freq )<train_on_grid>
def create_model9(embedding_matrix):
mdln="9-Bidirect(CuDNNGRU)-Bidirect(CuDNNGRU)-Bidirect(CuDNNGRU)-Attention-Dense"
input_layer = Input(shape=(maxlen,))
embedding_layer = Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],
weights=[embedding_matrix], trainable=Trainable )(input_layer)
 num_lstm2=int(num_lstm)
 num_lstm12=num_lstm2+int(num_lstm2/2)
 x = Bidirectional(CuDNNGRU(num_lstm, return_sequences=True))(embedding_layer)
 x = Bidirectional(CuDNNGRU(num_lstm12, return_sequences=True))(x)
 x = Bidirectional(CuDNNGRU(num_lstm2, return_sequences=True))(x)
 x = Attention(maxlen )(x)
 preds = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=input_layer, outputs=preds)
 model.compile(loss='binary_crossentropy', optimizer=opt, metrics=met)
 return model, mdln
def grad_boost(X, Y, X_test):
parameters = {'max_depth' : [2, 4, 10, 15],
'n_estimators' : [10, 50, 100],
'min_samples_split' : [5, 10, 15],
}
kfold = model_selection.KFold(n_splits=3, random_state=random_state, shuffle = True)
 model_GBC = GradientBoostingClassifier(random_state = 4)
 search_GBC = GridSearchCV(model_GBC, parameters, n_jobs = -1, cv = kfold, scoring = 'accuracy',verbose=1)
 search_GBC.fit(X, Y)
 predicted= search_GBC.predict(X_test)
 print("Best parameters are:", search_GBC.best_params_)
 print("Best accuracy achieved:",search_GBC.cv_results_['mean_test_score'].mean())
 return search_GBC.best_params_, model_GBC, search_GBC, predicted 
<compute_train_metric>
def create_model10(embedding_matrix):
mdln="10-Spatial-Bidirect(CuDNNGRU)-conv2-maxpool-attavr-average-concatenate-drop-dense"
input_layer = Input(shape=(maxlen,))
embedding_layer = Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],
weights=[embedding_matrix], trainable=Trainable )(input_layer)
 recurrent_units = 64
filter1_nums = 128
embedding_layer = SpatialDropout1D(rate_drop_spatial )(embedding_layer)
 rnn_1 = Bidirectional(CuDNNGRU(num_lstm, return_sequences=True))(embedding_layer)
 conv_2 = Conv1D(filter1_nums, 2, kernel_initializer="normal", padding="valid", activation="relu", strides=1 )(rnn_1)
 maxpool = GlobalMaxPooling1D()(conv_2)
 attn = AttentionWeightedAverage()(conv_2)
 average = GlobalAveragePooling1D()(conv_2)
 concatenated = concatenate([maxpool, attn, average], axis=1)
 x = Dropout(rate_drop_dense )(concatenated)
 x = Dense(num_dense, activation="relu" )(x)
 output_layer = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=input_layer, outputs=output_layer)
 adam_optimizer = optimizers.Adam(lr=1e-3, clipvalue=5, decay=1e-5)
 model.compile(loss='binary_crossentropy', optimizer=adam_optimizer, metrics=met)
 return model, mdln
param_GBC_onehot, model_GBC_onehot, search_GBC_onehot, predicted_cv_GBC_onehot = grad_boost(X_onehot, Y_onehot, X_test_onehot)
 param_GBC_lab, model_GBC_lab, search_GBC_lab, predicted_cv_GBC_lab = grad_boost(X_lab, Y_lab, X_test_lab)
 param_GBC_mean, model_GBC_mean, search_GBC_mean, predicted_cv_GBC_mean = grad_boost(X_mean, Y_mean, X_test_mean)
 param_GBC_freq, model_GBC_freq, search_GBC_freq, predicted_cv_GBC_freq = grad_boost(X_freq, Y_freq, X_test_freq )<train_model>
def create_model11(embedding_matrix):
mdln="11-Capsule"
input_layer = Input(shape=(maxlen,))
embedding_layer = Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],
weights=[embedding_matrix], trainable=Trainable )(input_layer)
 x = SpatialDropout1D(rate=rate_drop_spatial )(embedding_layer)
 x = Bidirectional(CuDNNGRU(num_lstm, return_sequences=True,
kernel_initializer=glorot_normal(seed=123000), recurrent_initializer=orthogonal(gain=1.0, seed=10000)) )(x)
 x = Capsule(num_capsule=10, dim_capsule=10, routings=4, share_weights=True )(x)
 x = Flatten()(x)
 x = Dense(num_dense, activation='relu', kernel_initializer=glorot_normal(seed=123000))(x)
 x = Dropout(rate_drop_dense )(x)
 x = BatchNormalization()(x)
 preds = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=input_layer, outputs=preds)
 model.compile(loss='binary_crossentropy', optimizer=Adam() ,)
 return model,mdln
def fit_pred_GBC(X, Y, X_test):
model_GBC = GradientBoostingClassifier(max_depth = 2, min_samples_split = 15, n_estimators = 10,\
random_state = 4, max_features= 'auto')
 model_GBC.fit(X, Y)
 predicted= model_GBC.predict(X_test)
 return predicted, model_GBC<predict_on_test>
def create_model12(embedding_matrix):
mdln="12-SpatialDropout+Bidirect(CudNNGRU)+Bidirect(CudNLSTM)+Attention+Attention+Dense+Dropout"
input_layer = Input(shape=(maxlen,))
embedding_layer = Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],
weights=[embedding_matrix], trainable=Trainable )(input_layer)
 x = SpatialDropout1D(rate_drop_spatial )(embedding_layer)
 x = Bidirectional(CuDNNLSTM(num_lstm, return_sequences=True))(x)
 y = Bidirectional(CuDNNGRU(num_lstm, return_sequences=True))(x)
 atten_1 = AttentionWeightedAverage()(x)
 atten_2 = AttentionWeightedAverage()(y)
 avg_pool = GlobalAveragePooling1D()(y)
 max_pool = GlobalMaxPooling1D()(y)
 conc = concatenate([atten_1, atten_2, avg_pool, max_pool])
 conc = Dense(num_dense, activation="relu" )(conc)
 conc = Dropout(rate_drop_dense )(conc)
 preds = Dense(1, activation="sigmoid" )(conc)
 model = Model(inputs=input_layer, outputs=preds)
 model.compile(loss='binary_crossentropy', optimizer=opt, metrics=met)
 return model,mdln
predicted_GBC_onehot, model_GBC_onehot = fit_pred_GBC(X_onehot, Y_onehot, X_test_onehot)
 predicted_GBC_lab, model_GBC_lab = fit_pred_GBC(X_lab, Y_lab, X_test_lab)
 predicted_GBC_mean, model_GBC_mean = fit_pred_GBC(X_mean, Y_mean, X_test_mean)
 predicted_GBC_freq, model_GBC_freq = fit_pred_GBC(X_freq, Y_freq, X_test_freq )<train_on_grid>
def create_model13(embedding_matrix):
mdln="13-SpatialDropout+Bidirect(CudNNGRU)+Bidirect(CudNLSTM)+Attention+Attention+Dense+Dropout"
input_layer = Input(shape=(maxlen,))
embedding_layer = Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],
weights=[embedding_matrix], trainable=Trainable )(input_layer)
 x = SpatialDropout1D(rate_drop_spatial )(embedding_layer)
 x = Bidirectional(CuDNNLSTM(num_lstm, kernel_initializer=glorot_uniform(seed = 2018), return_sequences=True))(x)
 y = Bidirectional(CuDNNGRU(num_lstm, kernel_initializer=glorot_uniform(seed = 2018),return_sequences=True))(x)
 atten_1 = Attention(maxlen )(x)
 atten_2 = Attention(maxlen )(y)
 avg_pool = GlobalAveragePooling1D()(y)
 max_pool = GlobalMaxPooling1D()(y)
 conc = concatenate([atten_1, atten_2, avg_pool, max_pool])
 conc = Dense(num_dense, kernel_initializer=he_uniform(seed=2018), activation="relu" )(conc)
 conc = Dropout(rate_drop_dense )(conc)
 preds = Dense(1, activation="sigmoid" )(conc)
 model = Model(inputs=input_layer, outputs=preds)
 model.compile(loss='binary_crossentropy', optimizer=opt, metrics=met)
 return model,mdln
def mod_KNN(X, Y, X_test):
model_KNN=make_pipeline(MinMaxScaler() ,KNeighborsClassifier())
 kfold = model_selection.KFold(n_splits=3, random_state=random_state, shuffle = True)
 parameters=[{'kneighborsclassifier__n_neighbors': [2,3,4,5,6,7,8,9,10]}]
search_KNN = GridSearchCV(estimator=model_KNN, param_grid=parameters, scoring='accuracy', cv=kfold)
 scores_KNN=cross_val_score(search_KNN, X, Y,scoring='accuracy', cv=kfold, verbose=1)
 search_KNN.fit(X, Y)
 predicted= search_KNN.predict(X_test)
 print("Best parameters are:", search_KNN.best_params_)
 print("Best accuracy achieved:",search_KNN.cv_results_['mean_test_score'].mean())
 return search_KNN.best_params_, model_KNN, search_KNN, predicted 
<compute_train_metric>
def create_model14(embedding_matrix):
mdln="14-Bidirect(CuDNNLSTM)-SpatialDropout-GlobalMax-BatchNorm-Dense-Dropout"
input_layer = Input(shape=(maxlen,))
embedding_layer = Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],
weights=[embedding_matrix], trainable=Trainable )(input_layer)
 x = CuDNNLSTM(num_lstm, return_sequences=True )(embedding_layer)
 x = SpatialDropout1D(rate_drop_spatial )(x)
 x = GlobalMaxPool1D()(x)
 x = BatchNormalization()(x)
 x = Dense(num_dense )(x)
 x = Dropout(rate_drop_dense )(x)
 preds = Dense(1, activation='sigmoid' )(x)
 model = Model(inputs=input_layer, outputs=preds)
 model.compile(loss='binary_crossentropy',optimizer=opt, metrics=met)
 return model, mdln
param_KNN_onehot, model_KNN_onehot, search_KNN_onehot, predicted_cv_KNN_onehot = mod_KNN(X_onehot, Y_onehot, X_test_onehot)
 param_KNN_lab, model_KNN_lab, search_KNN_lab, predicted_cv_KNN_lab = mod_KNN(X_lab, Y_lab, X_test_lab)
 param_KNN_mean, model_KNN_mean, search_KNN_mean, predicted_cv_KNN_mean = mod_KNN(X_mean, Y_mean, X_test_mean)
 param_KNN_freq, model_KNN_freq, search_KNN_freq, predicted_cv_KNN_freq = mod_KNN(X_freq, Y_freq, X_test_freq )<train_model>
def create_model15(embedding_matrix):
mdln="15-SpatialDropout-Bidirect(CudNNGRU)+Concat(GlobalMax+GlobalAverage)+Dense+Dropout"
input_layer = Input(shape=(maxlen,))
embedding_layer = Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],
weights=[embedding_matrix], trainable=Trainable )(input_layer)
 x = SpatialDropout1D(rate_drop_spatial )(embedding_layer)
 x = Bidirectional(CuDNNGRU(num_lstm, return_sequences=True))(x)
 avg_pool = GlobalAveragePooling1D()(x)
 max_pool = GlobalMaxPooling1D()(x)
 conc = concatenate([avg_pool, max_pool])
 x = Dense(num_dense, activation="relu" )(conc)
 x = Dropout(rate_drop_dense )(x)
 preds = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=input_layer, outputs=preds)
 model.compile(loss='binary_crossentropy',optimizer=opt, metrics=met)
 return model, mdln
def fit_pred_KNN(X, Y, X_test):
model_KNN = make_pipeline(MinMaxScaler() ,KNeighborsClassifier(n_neighbors=11))
model_KNN.fit(X, Y)
 predicted= model_KNN.predict(X_test)
 return predicted, model_KNN<predict_on_test>
def create_model16(embedding_matrix):
mdln="16-Bidirect(CudNNGRU)+GlobalMax+Dense+Dropout"
input_layer = Input(shape=(maxlen,))
embedding_layer = Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],
weights=[embedding_matrix], trainable=Trainable )(input_layer)
 x = Bidirectional(CuDNNLSTM(num_lstm, return_sequences=True))(embedding_layer)
 x = GlobalMaxPool1D()(x)
 x = Dense(num_dense, activation="relu" )(x)
 x = Dropout(rate_drop_dense )(x)
 preds = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=input_layer, outputs=preds)
 model.compile(loss='binary_crossentropy',optimizer=opt, metrics=met)
 return model, mdln
predicted_KNN_onehot, model_KNN_onehot = fit_pred_KNN(X_onehot, Y_onehot, X_test_onehot)
 predicted_KNN_lab, model_KNN_lab = fit_pred_KNN(X_lab, Y_lab, X_test_lab)
 predicted_KNN_mean, model_KNN_mean = fit_pred_KNN(X_mean, Y_mean, X_test_mean)
 predicted_KNN_freq, model_KNN_freq = fit_pred_KNN(X_freq, Y_freq, X_test_freq )<train_on_grid>
def create_model17(embedding_matrix):
mdln="17-Bidirec(CuDNNLSTM)-GlobalMax-Dense-Dropout-Dense-Dropout"
input_layer = Input(shape=(maxlen,))
embedding_layer = Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],
weights=[embedding_matrix], trainable=Trainable )(input_layer)
 x = Bidirectional(CuDNNLSTM(num_lstm, return_sequences=True))(embedding_layer)
 x = GlobalMaxPool1D()(x)
 x = Dense(num_dense, activation="relu" )(x)
 x = Dropout(rate_drop_dense )(x)
 x = Dense(num_dense, activation="relu" )(x)
 x = Dropout(rate_drop_dense )(x)
 preds = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=input_layer, outputs=preds)
 model.compile(loss='binary_crossentropy',optimizer=opt, metrics=met)
 return model, mdln
def mod_SVC(X, Y, X_test):
model_SVC=make_pipeline(StandardScaler() ,SVC(random_state=1))
parameters=[{'svc__C': [0.0001,0.001,0.1,1, 10, 100],
'svc__gamma':[0.0001,0.001,0.1,1,10,50,100],
'svc__kernel':['rbf'],
'svc__degree' : [1,2,3,4]
}]
kfold = model_selection.KFold(n_splits=3, random_state=random_state, shuffle = True)
 search_SVC = GridSearchCV(estimator=model_SVC, param_grid = parameters, scoring='accuracy', cv=kfold)
 scores_SVC=cross_val_score(search_SVC, X, Y,scoring='accuracy', cv=kfold, verbose =1)
 search_SVC.fit(X, Y)
 predicted= search_SVC.predict(X_test)
 print("Best parameters are:", search_SVC.best_params_)
 print("Best accuracy achieved:",search_SVC.cv_results_['mean_test_score'].mean())
 return search_SVC.best_params_, model_SVC, search_SVC, predicted<compute_train_metric>
def create_model18(embedding_matrix):
mdln="18-SpatialDropout+Bidirect(CudNNGRU)+Bidirect(CudNLSTM)+Attention+Attention+Dense+Dropout"
input_layer = Input(shape=(maxlen,))
embedding_layer = Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],
weights=[embedding_matrix], trainable=Trainable )(input_layer)
 x = SpatialDropout1D(rate_drop_spatial )(embedding_layer)
 x = Bidirectional(CuDNNLSTM(num_lstm, return_sequences=True))(x)
 y = Bidirectional(CuDNNGRU(num_lstm, return_sequences=True))(x)
 atten_1 = Attention(maxlen )(x)
 atten_2 = Attention(maxlen )(y)
 avg_pool = GlobalAveragePooling1D()(y)
 max_pool = GlobalMaxPooling1D()(y)
 conc = concatenate([atten_1, atten_2, avg_pool, max_pool])
 conc = Dense(num_dense, activation="relu" )(conc)
 conc = Dropout(rate_drop_dense )(conc)
 preds = Dense(1, activation="sigmoid" )(conc)
 model = Model(inputs=input_layer, outputs=preds)
 model.compile(loss='binary_crossentropy', optimizer=opt, metrics=met)
 return model,mdln
param_SVC_onehot, model_SVC_onehot, search_SVC_onehot, predicted_cv_SVC_onehot = mod_SVC(X_onehot, Y_onehot, X_test_onehot)
 param_SVC_lab, model_SVC_lab, search_SVC_lab, predicted_cv_SVC_lab = mod_SVC(X_lab, Y_lab, X_test_lab)
 param_SVC_mean, model_SVC_mean, search_SVC_mean, predicted_cv_SVC_mean = mod_SVC(X_mean, Y_mean, X_test_mean)
 param_SVC_freq, model_SVC_freq, search_SVC_freq, predicted_cv_SVC_freq = mod_SVC(X_freq, Y_freq, X_test_freq )<train_model>
def create_model19(embedding_matrix):
mdln="19-SpatialDropout+Bidirect(CudNNGRU)+Conv1D+Concat(GlobalMax+GlobalAverage)+Dense+Dropout"
input_layer = Input(shape=(maxlen,))
embedding_layer = Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],
weights=[embedding_matrix], trainable=Trainable )(input_layer)
 x = SpatialDropout1D(rate_drop_spatial )(embedding_layer)
 x = Bidirectional(CuDNNGRU(num_lstm, return_sequences=True))(x)
 x = Conv1D(64, kernel_size = 3, padding = "valid", kernel_initializer = "glorot_uniform" )(x)
 tower_1 = GlobalMaxPool1D()(x)
 tower_2 = GlobalAveragePooling1D()(x)
 output = concatenate([ tower_1, tower_2])
 out = Dense(num_dense, activation="relu" )(output)
 out = Dropout(rate_drop_dense )(out)
 preds = Dense(1, activation="sigmoid" )(out)
 model = Model(inputs=input_layer, outputs=preds)
 model.compile(loss='binary_crossentropy',optimizer=Adam(lr=1e-3),metrics=met)
 return model, mdln
def fit_pred_SVC(X, Y, X_test):
model_SVC = make_pipeline(StandardScaler() ,SVC(random_state=random_state, C= 1, gamma = 0.001, kernel = 'rbf', degree =1))
model_SVC.fit(X, Y)
 predicted= model_SVC.predict(X_test)
 return predicted, model_SVC<predict_on_test>
def create_model(label,embedding_matrix):
if(label=="0"):
return create_model0(embedding_matrix)
 if(label=="1"):
return create_model1(embedding_matrix)
 if(label=="2"):
return create_model2(embedding_matrix)
 if(label=="3"):
return create_model3(embedding_matrix)
 if(label=="4"):
return create_model4(embedding_matrix)
 if(label=="5"):
return create_model5(embedding_matrix)
 if(label=="6"):
return create_model6(embedding_matrix)
 if(label=="7"):
return create_model7(embedding_matrix)
 if(label=="8"):
return create_model8(embedding_matrix)
 if(label=="9"):
return create_model9(embedding_matrix)
 if(label=="10"):
return create_model10(embedding_matrix)
 if(label=="11"):
return create_model11(embedding_matrix)
 if(label=="12"):
return create_model12(embedding_matrix)
 if(label=="13"):
return create_model13(embedding_matrix)
 if(label=="14"):
return create_model14(embedding_matrix)
 if(label=="15"):
return create_model15(embedding_matrix)
 if(label=="16"):
return create_model16(embedding_matrix)
 if(label=="17"):
return create_model17(embedding_matrix)
 if(label=="18"):
return create_model18(embedding_matrix)
 if(label=="19"):
return create_model19(embedding_matrix)
 return None,"None"
predicted_SVC_onehot, model_SVC_onehot = fit_pred_SVC(X_onehot, Y_onehot, X_test_onehot)
 predicted_SVC_lab, model_SVC_lab = fit_pred_SVC(X_lab, Y_lab, X_test_lab)
 predicted_SVC_mean, model_SVC_mean = fit_pred_SVC(X_mean, Y_mean, X_test_mean)
 predicted_SVC_freq, model_SVC_freq = fit_pred_SVC(X_freq, Y_freq, X_test_freq )<compute_test_metric>
def best_F1(v_y ,t_y,show_plot=False):
bs=0
bt=0
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 score = metrics.f1_score(v_y,(t_y>=thresh ).astype(int))
if score >= bs:
bt = thresh
bs = score
if(show_plot):
print("
Best F1 score at threshold %2.4f is %2.4f
" %(bt, bs))
plot_confusion_matrix(v_y, np.array(pd.Series(t_y.reshape(-1,)).map(lambda x:1 if x>=bt else 0)))
 d=confusion_matrix(v_y,(t_y>=bt ).astype(int))
print("Total:",d.sum() ,"/Insincere:",d[1,:].sum() ,"/Sincere:",d[0,:].sum())
 print("Total Erros:",(d[0,1]+d[1,0]),"/Insincere: %2f"%(( d[0,1]/(d[0,1]+d[1,1]))),"/Sincere: %2f" %(( d[1,0]/(d[1,0]+d[0,0]))))
return bt, bs
predicted = np.where(((predicted_SVC_mean + predicted_KNN_onehot+predicted_RFC_onehot+predicted_RFC_freq+ predicted_RFC_mean)/5)> 0.5, 1, 0)
<save_to_csv>
def f1(y_true, y_pred):
def recall(y_true, y_pred):

 true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
 recall = true_positives /(possible_positives + K.epsilon())
 return recall
def precision(y_true, y_pred):

 true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
 precision = true_positives /(predicted_positives + K.epsilon())
 return precision
precision = precision(y_true, y_pred)
 recall = recall(y_true, y_pred)
 return 2*(( precision*recall)/(precision+recall+K.epsilon()))
test =pd.read_csv(".. /input/test.csv")
 submission = pd.DataFrame({'PassengerId': test['PassengerId'],'Survived':predicted})
 submission.head()
filename = 'Titanic Predictions Public.csv'
submission.to_csv(filename,index=False)
 print('Saved file: ' + filename )<set_options>
def rec_results(Model,Embedding,Thresh,Score,StartTime):
global resultsdf
resultsdf = resultsdf.append({'Model':Model,
'Method':method,
'Embedding':Embedding,
'Pretext':pretext_proc,
'Weigth':wgt,
'F1':Score,
"Threshold":Thresh,
"Duration":str(round(( time.time() -StartTime)/60,0)) ,
'Epochs':epochs,
'EpochsUsed':epoc_used,
'Trainable':Trainable,
'Opt':opt,
'MaxLength':maxlen,
'MaxFeatures':max_features,
'batch_size':batch_size,
'patience':patience,
'num_lstm':num_lstm,
'rate_drop_lstm':rate_drop_lstm,
'num_dense':num_dense,
'rate_drop_dense':rate_drop_dense,
'rate_drop_spatial':rate_drop_spatial,
'opt':opt,
'met':met,
'es_mon':es_mon,
'es_mode':es_mode,
'Date':datetime.now().strftime("%d-%m-%Y %H:%M")}, ignore_index=True )
pd.plotting.register_matplotlib_converters()
%matplotlib inline
sns.set_style('dark')
 print('Setup complete' )<load_from_csv>
def load_embedding(emb,word_index):
estart = time.time()
print('Indexing '+emb+' vectors')
 if(emb=="Glove"):
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
if(emb=="Google"):
EMBEDDING_FILE = '.. /input/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin'
embeddings_index = KeyedVectors.load_word2vec_format(EMBEDDING_FILE, binary=True)
 if(emb=="Paragram"):
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 if(emb=="Wiki"):
EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100)
 print("Vector",EMBEDDING_FILE)
 print("End Indexing:",(str(timedelta(seconds=(time.time() -estart)))))
 estart = time.time()
print('Preparing embedding matrix')
 out_of_features=0
out_of_embedding=0
in_embedding=0
if(emb=="Wiki" or emb=="Glove" or emb=="Paragram"):
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index)) +1
np.random.seed(2018)
 embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features:
out_of_features+=1
continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
in_embedding+=1
else:
out_of_embedding+=1
else:
embed_size=300
nb_words = min(max_features, len(word_index)) +1
np.random.seed(2018)
 embedding_matrix=(np.random.rand(nb_words, embed_size)- 0.5)/ 5.0
for word, i in word_index.items() :
if i >= max_features:
out_of_features+=1
continue
if word in embeddings_index:
embedding_vector = embeddings_index.get_vector(word)
 embedding_matrix[i] = embedding_vector
in_embedding+=1
else:
out_of_embedding+=1
print("Total Vocabulary:",len(word_index))
print("Out of Features:", out_of_features)
 print("Out of Embeddings:",out_of_embedding)
 print("Mapped Embeddings:",in_embedding)
 print("End Preparing embedding matrix:",(str(timedelta(seconds=(time.time() -estart)))))
 del embeddings_index
return embedding_matrix
train_data = pd.read_csv('/kaggle/input/titanic/train.csv')
 train_data.head()<load_from_csv>
def evaluate_model1(label,emb, embedding_matrix,train_X,train_y,t_X,t_y,v_X,v_y, epochs, batch_size, patience=0):
global resultsdf
global pred_val_y
global prev_test_y
global met
global epoc_used
best_thresh = 0.0
best_score = 0.0
mstart = time.time()
model=None
model,mdln = create_model(label,embedding_matrix)
 print("")
 print(mdln+'-'+emb)
 model_checkpoint = ModelCheckpoint('./model.hdf5', monitor=es_mon, mode=es_mode,
verbose=True, save_best_only=True,
save_weights_only=False)
 early_stopping = EarlyStopping(monitor=es_mon, mode=es_mode, patience=patience,verbose=True)
 callbacks = [ early_stopping, model_checkpoint]
print("Model Fitting:")
 hist= model.fit(t_X, t_y, batch_size=batch_size, epochs=epochs,
shuffle=True, verbose=True, validation_data=(v_X, v_y),
callbacks=callbacks)
 ",hist.history.keys())
 epoc_used=len(hist.history[es_mon])
 model.load_weights('./model.hdf5')
 print("Predicting Values:")
 pvy = model.predict([v_X], batch_size=batch_size, verbose=2)
 best_thresh,best_score=best_F1(v_y,pvy[:,0] , False)
 pty = model.predict([test_X], batch_size=1024, verbose=2)
 rec_results(mdln,emb,best_thresh,best_score,mstart)
 del model
gc.collect()
time.sleep(10)
 return pvy[:,0],pty[:,0],best_score,best_thresh
test_data = pd.read_csv('/kaggle/input/titanic/test.csv')
 test_data.head()<count_values>
def evaluate_model2(label,emb, embedding_matrix,train_X,train_y, t_X, t_y, v_X, v_y, epochs, batch_size, patience=0):
global resultsdf
global pred_val_y
global prev_test_y
global met
best_thresh = 0.0
best_score = 0.0
mstart = time.time()
model=None
model,mdln = create_model(label, embedding_matrix)
 print("")
 print(mdln+'-'+emb)
 bs=0
pt=0
print("Model Fitting:")
 for e in range(epochs):
model.fit(t_X, t_y, batch_size=batch_size, epochs=1, validation_data=(v_X, v_y))
print("Predicting Values:")
 pvy = model.predict([v_X], batch_size=1024, verbose=0)
 best_thresh,best_score=best_F1(v_y,pvy[:,0], False)
 print("epoch ",e+1,"/",epochs)
 if(best_score > bs):
print("F1 Score Improved from %2.4f to %2.4f" %(bs,best_score))
bs = best_score
bt = best_thresh
epoc_used=e+1
model.save_weights('./model.hdf5')
 else:
print("F1 Score not improved")
 pt=pt+1
if(pt>patience):
break
model.load_weights('./model.hdf5')
 pty = model.predict([test_X], batch_size=1024, verbose=0)
 rec_results(mdln,emb,bt,bs,mstart)
 del model
gc.collect()
time.sleep(10)
 return pvy[:,0],pty[:,0],bs,bt
train_data['Survived'].value_counts(normalize=True )<groupby>
def evaluate_model3(label,emb, embedding_matrix,train_X,train_y, t_X, t_y, v_X, v_y, epochs, batch_size, patience=0):
global resultsdf
global pred_val_y
global prev_test_y
global met
global clr
global epoc_used
clr = CyclicLR(base_lr=0.001, max_lr=0.01,
step_size=300., mode='exp_range',
gamma=0.99994)
 callback = [clr,]
best_thresh = 0.0
best_score = 0.0
mstart = time.time()
model=None
model,mdln = create_model(label, embedding_matrix)
 print("")
 print(mdln+'-'+emb)
 bs=0
pt=0
print("Model Fitting:")
 for e in range(epochs):
model.fit(t_X, t_y, batch_size=batch_size, epochs=1, validation_data=(v_X, v_y),callbacks = callback,verbose=2)
 print("Predicting Values:")
 pvy = model.predict([v_X], batch_size=1024, verbose=0)
 print("F1 - Score:")
 best_thresh,best_score=best_F1(v_y,pvy[:,0])
 print("epoch ",e+1,"/",epochs)
 if(best_score > bs):
print("F1 Score Improved from %2.4f to %2.4f" %(bs,best_score))
bs = best_score
bt = best_thresh
epoc_used=e+1
model.save_weights('./model.hdf5')
 else:
print("F1 Score not improved")
 pt=pt+1
if(pt>patience):
break
model.load_weights('./model.hdf5')
 pty = model.predict([test_X], batch_size=1024, verbose=0)
 rec_results(mdln,emb,bt,bs,mstart)
 del model
gc.collect()
time.sleep(10)
 return pvy[:,0],pty[:,0],bs,bt
train_data.groupby('Pclass' ).Survived.mean()<groupby>
def evaluate_model4(label,emb, embedding_matrix,train_X,train_y,t_X,t_y,v_X,v_y, epochs, batch_size, patience=0):
global resultsdf
global pred_val_y
global prev_test_y
global met
global clr
global epoc_used
clr = CyclicLR(base_lr=0.001, max_lr=0.01,
step_size=300., mode='exp_range',
gamma=0.99994)
 best_thresh = 0.0
best_score = 0.0
mstart = time.time()
model=None
model,mdln = create_model(label,embedding_matrix)
 print("")
 print(mdln+'-'+emb)
 model_checkpoint = ModelCheckpoint('./model.hdf5', monitor=es_mon, mode=es_mode,
verbose=True, save_best_only=True,
save_weights_only=False)
 early_stopping = EarlyStopping(monitor=es_mon, mode=es_mode, patience=patience,verbose=True)
 callbacks = [ early_stopping, model_checkpoint,clr,]
print("Model Fitting:")
 hist= model.fit(t_X, t_y, batch_size=batch_size, epochs=epochs,
shuffle=True, verbose=True, validation_data=(v_X, v_y),
callbacks=callbacks)
 epoc_used=len(hist.history[es_mon])
 print("Getting the Best Model")
 model.load_weights('./model.hdf5')
 print("Predicting Values:")
 pvy = model.predict([v_X], batch_size=batch_size, verbose=2)
 best_thresh,best_score=best_F1(v_y,pvy[:,0], False)
 pty = model.predict([test_X], batch_size=1024, verbose=2)
 rec_results(mdln,emb,best_thresh,best_score,mstart)
 del model
gc.collect()
time.sleep(10)
 return pvy[:,0],pty[:,0],best_score,best_thresh
train_data.groupby(['Pclass', 'Sex'] ).Survived.mean()<filter>
def evaluate_model5(label,emb, embedding_matrix,train_X,train_y, t_X, t_y, v_X, v_y, epochs, batch_size, patience=0):
global resultsdf
global pred_val_y
global prev_test_y
global met
global clr
global epoc_used
clr = CyclicLR(base_lr=0.001, max_lr=0.01,
step_size=300., mode='exp_range',
gamma=0.99994)
 callback = [clr,]
best_thresh = 0.0
best_score = 0.0
mstart = time.time()
model=None
model,mdln = create_model(label, embedding_matrix)
 print("")
 print(mdln+'-'+emb)
 bs=0
pt=0
print("Model Fitting:")
 for e in range(epochs):
model.fit(t_X, t_y, batch_size=batch_size, epochs=1, validation_data=(v_X, v_y),verbose=2)
 print("Predicting Values:")
 pvy = model.predict([v_X], batch_size=1024, verbose=0)
 best_thresh = 0.0
best_score = 0.0
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 score = metrics.f1_score(v_y,(pvy[:,0] > thresh ).astype(int))
if score > best_score:
print("F1COnf Score Improved from %2.4f to %2.4f" %(best_score,score))
best_thresh = thresh
best_score = score
best_thresh = 0.0
best_score = 0.0
print("F1 - Score:")
 best_thresh,best_score=best_F1(v_y,pvy[:,0])
 print("epoch ",e+1,"/",epochs)
 epoc_used=e+1
if(best_score > bs):
print("F1 Score Improved from %2.4f to %2.4f" %(bs,best_score))
bs = best_score
bt = best_thresh
pty = model.predict([test_X], batch_size=1024, verbose=0)
 rec_results(mdln,emb,bt,bs,mstart)
 del model
gc.collect()
time.sleep(10)
 return pvy[:,0],pty[:,0],bs,bt
train_data.loc[train_data.Fare==0]<prepare_x_and_y>
def evaluate_model6(label,emb, embedding_matrix,train_X,train_y,t_X,t_y,v_X,v_y, epochs, batch_size, patience=0):
global resultsdf
global pred_val_y
global prev_test_y
global met
global epoc_used
best_thresh = 0.0
best_score = 0.0
mstart = time.time()
model=None
model,mdln = create_model(label,embedding_matrix)
 print("")
 print(mdln+'-'+emb)
 model_checkpoint = ModelCheckpoint('./model.hdf5', monitor=es_mon, mode=es_mode,
verbose=True, save_best_only=True,
save_weights_only=False)
 reduce_lr = ReduceLROnPlateau(monitor=es_mon, factor=0.5, patience=1, min_lr=0.0001, verbose=True)
 early_stopping = EarlyStopping(monitor=es_mon,min_delta=0.0001, mode='auto', patience=patience,verbose=True)
 callbacks = [ early_stopping,model_checkpoint,reduce_lr]
print("Model Fitting:")
 hist= model.fit(t_X, t_y, batch_size=batch_size, epochs=epochs,
shuffle=True, verbose=True, validation_data=(v_X, v_y),
callbacks=callbacks)
 ",hist.history.keys())
 epoc_used=len(hist.history[es_mon])
 model.load_weights('./model.hdf5')
 print("Predicting Values:")
 pvy = model.predict([v_X], batch_size=batch_size, verbose=2)
 best_thresh,best_score=best_F1(v_y,pvy[:,0] , False)
 pty = model.predict([test_X], batch_size=1024, verbose=2)
 rec_results(mdln,emb,best_thresh,best_score,mstart)
 del model
gc.collect()
time.sleep(10)
 return pvy[:,0],pty[:,0],best_score,best_thresh
def remove_zero_fares(row):
if row.Fare == 0:
row.Fare = np.NaN
return row
train_data = train_data.apply(remove_zero_fares, axis=1)
 test_data = test_data.apply(remove_zero_fares, axis=1)
 print('Number of zero-Fares: {:d}'.format(train_data.loc[train_data.Fare==0].shape[0]))<feature_engineering>
def evaluate_stratifiedkfold(label,mt,emb,train_X,train_y, test_X, n_splits, stop_split, epochs, batch_size,patience):
global resultsdf
global allpred_val_y
global allprev_test_y
global mn
global method
method=mt
random_seed = 2018
train_stratified=np.zeros([len(train_X),(stop_split)])
 test_stratified = np.zeros([len(test_X),(stop_split)])
 msstart = time.time()
embedding_matrix=embedding_matrix_combined
if(emb=="Glove"):
embedding_matrix=embedding_matrix_1
if(emb=="Wiki"):
embedding_matrix=embedding_matrix_2
if(emb=="Paragram"):
embedding_matrix=embedding_matrix_3
if(emb=="Google"):
embedding_matrix=embedding_matrix_4
if(emb=="Concatenated"):
embedding_matrix=embedding_matrix_concatenated
splits = list(StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=random_seed ).split(train_X, train_y))
mn=mn+1
bs=np.zeros([stop_split])
 bt=np.zeros([stop_split])
 for idx,(train_idx, valid_idx)in enumerate(splits):
K.clear_session()
X_split = train_X[train_idx]
y_split = train_y[train_idx]
print("Split:",idx)
 print("Total X-train:",len(y_split)," Insincere:",y_split.sum() ,y_split.sum() / len(y_split))
X_val = train_X[valid_idx]
y_val = train_y[valid_idx]
print("Total X-val:",len(y_val)," Insincere:",y_val.sum() , y_val.sum() /len(y_val))
if(mt=="0"):
pred_train_y, pred_test_y, best_score, best_thresh=evaluate_model(md,emb,embedding_matrix,train_X,train_y, X_split, y_split, X_val, y_val, epochs, batch_size ,patience)
 if(mt=="1"):
pred_train_y, pred_test_y, best_score, best_thresh=evaluate_model1(md,emb,embedding_matrix,train_X,train_y, X_split, y_split, X_val, y_val,epochs, batch_size ,patience)
 if(mt=="2"):
pred_train_y, pred_test_y, best_score, best_thresh=evaluate_model2(md,emb,embedding_matrix,train_X,train_y, X_split, y_split, X_val, y_val,epochs, batch_size ,patience)
 if(mt=="3"):
pred_train_y, pred_test_y, best_score, best_thresh=evaluate_model3(md,emb,embedding_matrix,train_X,train_y, X_split, y_split, X_val, y_val,epochs, batch_size ,patience)
 if(mt=="4"):
pred_train_y, pred_test_y, best_score, best_thresh=evaluate_model4(md,emb,embedding_matrix,train_X,train_y, X_split, y_split, X_val, y_val,epochs, batch_size ,patience)
 if(mt=="5"):
pred_train_y, pred_test_y, best_score, best_thresh=evaluate_model5(md,emb,embedding_matrix,train_X,train_y, X_split, y_split, X_val, y_val,epochs, batch_size ,patience)
 if(mt=="6"):
pred_train_y, pred_test_y, best_score, best_thresh=evaluate_model6(md,emb,embedding_matrix,train_X,train_y, X_split, y_split, X_val, y_val,epochs, batch_size ,patience)
 if(best_score > bs[idx]):
print("F1 Score Improved from %2.4f to %2.4f" %(bs[idx],best_score))
bs[idx] = best_score
bt[idx] = best_thresh
test_stratified[:,idx] = pred_test_y
if(( idx+1)==stop_split):
break
train_pred = train_stratified.mean(axis=1)
 test_pred = test_stratified.mean(axis=1)
 if(stop_split > 1):
rec_results("Model Mean:"+label+"-skfold- "+str(n_splits)+"Stop: "+str(stop_split),emb,bt.min() ,bs.mean() ,msstart)
 return train_pred,test_pred,bs.max() ,bt.min()
train_data['Title'] = train_data['Name'].apply(lambda x: x.split(',')[1].split('.')[0].strip())
 test_data['Title'] = test_data['Name'].apply(lambda x: x.split(',')[1].split('.')[0].strip() )<count_values>
embedding_matrix_1 = load_embedding("Glove",word_index)
 embedding_matrix_3 = load_embedding("Paragram",word_index)
 embedding_matrix_combined = np.mean([embedding_matrix_1, embedding_matrix_3], axis = 0)
 np.shape(embedding_matrix_combined)
 embedding_matrix_concatenated = np.concatenate(( embedding_matrix_1, embedding_matrix_3), axis=1)
 np.shape(embedding_matrix_concatenated )
train_data['Title'].value_counts()<count_values>
set_all_parameters()
resultsdf = pd.DataFrame(columns=['Model','F1','Embedding','Pretext','Weigth','Duration','Method',
'Epochs','EpochsUsed','patience','batch_size'
,'MaxLength','MaxFeatures' ])
 allpred_train_y=np.zeros([len(train_X),(len(model_list)) ])
 allpred_test_y=np.zeros([len(test_X),(len(model_list)) ])
 allbest=np.zeros([(len(model_list)) ])
 allthresh=np.zeros([(len(model_list)) ])
 epoc_used=0
mn=-1
for ne,item in enumerate(model_list):
md=item[0]
wgt=item[1]
mt=item[2]
emb=item[3]
epochs=item[4]
patience=item[5]
batch_size=item[6]
nsplits=item[7]
stop_split=item[8]
num_lstm=item[9]
num_dense=item[10]
rate_drop_lstm = item[11]
rate_drop_dense = item[12]
rate_drop_spatial=item[13]
loss=item[14]
opt=item[15]
met=item[16]
es_mon=item[17]
es_mode=item[18]
print(mt)
 print("md-"+md)
 trainp,testp,bs,bt=evaluate_stratifiedkfold(md,mt,emb,train_X,train_y, test_X, nsplits, stop_split, epochs, batch_size,patience)
 allbest[mn]=bs
allthresh[mn]=bt
allpred_test_y[:,mn] = testp
print("FIM-"+md)
test_data['Title'].value_counts()<categorify>
twgt=0
wgt_test_y=0
wgt_train_y=0
essemblet="Essemble: "
for ne,item in enumerate(model_list):
wg=item[1]
twgt=twgt+wg
wgt_test_y =wgt_test_y +(allpred_test_y[:,ne]*wg)
 essemblet=essemblet+item[0]+" "
wgt_test_y=wgt_test_y/twgt
train_data['Title'].replace(['Mme', 'Ms', 'Lady', 'Mlle', 'the Countess', 'Dona'], 'Miss', inplace=True)
 test_data['Title'].replace(['Mme', 'Ms', 'Lady', 'Mlle', 'the Countess', 'Dona'], 'Miss', inplace=True)
 train_data['Title'].replace(['Major', 'Col', 'Capt', 'Don', 'Sir', 'Jonkheer'], 'Mr', inplace=True)
 test_data['Title'].replace(['Major', 'Col', 'Capt', 'Don', 'Sir', 'Jonkheer'], 'Mr', inplace=True )<groupby>
print(allthresh)
 print(allthresh.min())
 print(allthresh.mean())
 y_te =(np.array(wgt_test_y)>= allthresh.min() ).astype(np.int)
 submit_df = pd.DataFrame({"qid": test_df["qid"], "prediction": y_te})
 submit_df.to_csv("./submission.csv",index=False)
 rec_results('Time Elapsed','',0,0,bstart)
 display(resultsdf )