Kaggle competition: Titanic - Machine Learning from Disaster
puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…',
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─',
'▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]
def clean_text(x):
x = str(x)
 for punct in puncts:
if punct in x:
x = x.replace(punct, f' {punct} ')
 return x
def clean_numbers(x):
x = re.sub('[0-9]{5,}', '
x = re.sub('[0-9]{4}', '
x = re.sub('[0-9]{3}', '
x = re.sub('[0-9]{2}', '
return x
def _get_mispell(mispell_dict):
mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))
return mispell_dict, mispell_re
mispell_dict = {"aren't" : "are not",
"can't" : "cannot",
"couldn't" : "could not",
"didn't" : "did not",
"doesn't" : "does not",
"don't" : "do not",
"hadn't" : "had not",
"hasn't" : "has not",
"haven't" : "have not",
"he'd" : "he would",
"he'll" : "he will",
"he's" : "he is",
"i'd" : "I would",
"i'd" : "I had",
"i'll" : "I will",
"i'm" : "I am",
"isn't" : "is not",
"it's" : "it is",
"it'll":"it will",
"i've" : "I have",
"let's" : "let us",
"mightn't" : "might not",
"mustn't" : "must not",
"shan't" : "shall not",
"she'd" : "she would",
"she'll" : "she will",
"she's" : "she is",
"shouldn't" : "should not",
"that's" : "that is",
"there's" : "there is",
"they'd" : "they would",
"they'll" : "they will",
"they're" : "they are",
"they've" : "they have",
"we'd" : "we would",
"we're" : "we are",
"weren't" : "were not",
"we've" : "we have",
"what'll" : "what will",
"what're" : "what are",
"what's" : "what is",
"what've" : "what have",
"where's" : "where is",
"who'd" : "who would",
"who'll" : "who will",
"who're" : "who are",
"who's" : "who is",
"who've" : "who have",
"won't" : "will not",
"wouldn't" : "would not",
"you'd" : "you would",
"you'll" : "you will",
"you're" : "you are",
"you've" : "you have",
"'re": " are",
"wasn't": "was not",
"we'll":" will",
"didn't": "did not"}
mispellings, mispellings_re = _get_mispell(mispell_dict)
 def replace_typical_misspell(text):
def replace(match):
return mispellings[match.group(0)]
return mispellings_re.sub(replace, text )<load_from_csv>
warnings.filterwarnings("ignore")
def load_and_prec() :
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 print("Train shape : ",train_df.shape)
 print("Test shape : ",test_df.shape)
 train_df["question_text"] = train_df["question_text"].progress_apply(lambda x: clean_text(x))
test_df["question_text"] = test_df["question_text"].progress_apply(lambda x: clean_text(x))
train_df["question_text"] = train_df["question_text"].progress_apply(lambda x: clean_numbers(x))
test_df["question_text"] = test_df["question_text"].progress_apply(lambda x: clean_numbers(x))
train_df["question_text"] = train_df["question_text"].progress_apply(lambda x: replace_typical_misspell(x))
test_df["question_text"] = test_df["question_text"].progress_apply(lambda x: replace_typical_misspell(x))
train_X = train_df["question_text"].fillna("_
splits = list(StratifiedKFold(n_splits=10,random_state=2018 ).split(train_X,train_df['target'].values))
test_X = test_df["question_text"].fillna("_
train_X, val_X = train_X[splits[0][0]],train_X[splits[0][1]]
train_y = train_df['target'].values[splits[0][0]]
val_y = train_df['target'].values[splits[0][1]]
tokenizer = Tokenizer(num_words=max_features, filters='', lower=True)
 tokenizer.fit_on_texts(list(train_X)+list(test_X))
train_X = tokenizer.texts_to_sequences(train_X)
 val_X = tokenizer.texts_to_sequences(val_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 train_X = pad_sequences(train_X, maxlen=maxlen)
 val_X = pad_sequences(val_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 np.random.seed(SEED)
 trn_idx = np.random.permutation(len(train_X))
val_idx = np.random.permutation(len(val_X))
train_X = train_X[trn_idx]
val_X = val_X[val_idx]
train_y = train_y[trn_idx]
val_y = val_y[val_idx]
return train_X, val_X, test_X, train_y, val_y, tokenizer.word_index 
<load_pretrained>
train=pd.read_csv('.. /input/titanic/train.csv')
 test=pd.read_csv('.. /input/titanic/test.csv')
 y_test=pd.read_csv('.. /input/titanic/gender_submission.csv' )
def load_glove(word_index):
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
emb_mean,emb_std = -0.005838499,0.48782197
embed_size = 300
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
with open(EMBEDDING_FILE, 'r', encoding="utf8")as f:
for line in f:
word, vec = line.split(' ', 1)
 if word not in word_index:
continue
i = word_index[word]
if i >= nb_words:
continue
embedding_vector = np.asarray(vec.split(' '), dtype='float32')[:300]
if len(embedding_vector)== 300:
embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_fasttext(word_index):
EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: break
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_para(word_index):
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
emb_mean,emb_std = -0.0053247833,0.49346462
embed_size = 300
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
with open(EMBEDDING_FILE, 'r', encoding="utf8", errors='ignore')as f:
for line in f:
word, vec = line.split(' ', 1)
 if word not in word_index:
continue
i = word_index[word]
if i >= nb_words:
continue
embedding_vector = np.asarray(vec.split(' '), dtype='float32')[:300]
if len(embedding_vector)== 300:
embedding_matrix[i] = embedding_vector
return embedding_matrix<compute_train_metric>
p=train.loc[train['Survived']==1 ]
print(len(p))
male=p.loc[p['Sex']=='male']
female=p.loc[p['Sex']=='female']
print(male['Pclass'].value_counts())
 print(female['Pclass'].value_counts() )
def threshold_search(y_true, y_proba):
best_threshold = 0
best_score = 0
for threshold in [i * 0.01 for i in range(100)]:
score = f1_score(y_true=y_true, y_pred=y_proba > threshold)
 if score > best_score:
best_threshold = threshold
best_score = score
rocauc = roc_auc_score(y_true, y_proba)
 p, r, _ = precision_recall_curve(y_true, y_proba)
 prauc = auc(r, p)
 search_result = {'threshold': best_threshold, 'f1': best_score, 'rocauc': rocauc, 'prauc': prauc}
return search_result<init_hyperparams>
p=train.loc[train['Survived']==0 ]
male=p.loc[p['Sex']=='male']
female=p.loc[p['Sex']=='female']
print(male['Pclass'].value_counts())
 print(female['Pclass'].value_counts() )
class CyclicLR(Callback):

 def __init__(self, base_lr=0.001, max_lr=0.006, step_size=2000., mode='triangular',
gamma=1., scale_fn=None, scale_mode='cycle'):
super(CyclicLR, self ).__init__()
self.base_lr = base_lr
self.max_lr = max_lr
self.step_size = step_size
self.mode = mode
self.gamma = gamma
if scale_fn == None:
if self.mode == 'triangular':
self.scale_fn = lambda x: 1.
 self.scale_mode = 'cycle'
elif self.mode == 'triangular2':
self.scale_fn = lambda x: 1/(2.**(x-1))
self.scale_mode = 'cycle'
elif self.mode == 'exp_range':
self.scale_fn = lambda x: gamma**(x)
 self.scale_mode = 'iterations'
else:
self.scale_fn = scale_fn
self.scale_mode = scale_mode
self.clr_iterations = 0.
 self.trn_iterations = 0.
 self.history = {}
self._reset()
def _reset(self, new_base_lr=None, new_max_lr=None,
new_step_size=None):

 if new_base_lr != None:
self.base_lr = new_base_lr
if new_max_lr != None:
self.max_lr = new_max_lr
if new_step_size != None:
self.step_size = new_step_size
self.clr_iterations = 0.
 def clr(self):
cycle = np.floor(1+self.clr_iterations/(2*self.step_size))
x = np.abs(self.clr_iterations/self.step_size - 2*cycle + 1)
 if self.scale_mode == 'cycle':
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(cycle)
 else:
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(self.clr_iterations)
 def on_train_begin(self, logs={}):
logs = logs or {}
if self.clr_iterations == 0:
K.set_value(self.model.optimizer.lr, self.base_lr)
 else:
K.set_value(self.model.optimizer.lr, self.clr())
 def on_batch_end(self, epoch, logs=None):
logs = logs or {}
self.trn_iterations += 1
self.clr_iterations += 1
self.history.setdefault('lr', [] ).append(K.get_value(self.model.optimizer.lr))
self.history.setdefault('iterations', [] ).append(self.trn_iterations)
 for k, v in logs.items() :
self.history.setdefault(k, [] ).append(v)
 K.set_value(self.model.optimizer.lr, self.clr() )<choose_model_class>
feature_mix=[]
for i in range(0,len(train)) :
marks=0
if(train['Sex'].iloc[i]=='female'):
marks=marks+10
if(train['Pclass'].iloc[i]!=1):
marks=marks+5
else:
marks=marks+2
if(train['Age'].iloc[i]<35 and train['Age'].iloc[i]>20):
marks=marks+4
else:
marks=marks+2
feature_mix.append(marks)
 train['feature_mix']=feature_mix
train['feature_mix'].value_counts()
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim<choose_model_class>
feature_mix=[]
for i in range(0,len(test)) :
marks=0
if(test['Sex'].iloc[i]=='female'):
marks=marks+10
if(test['Pclass'].iloc[i]!=1):
marks=marks+5
else:
marks=marks+2
if(test['Age'].iloc[i]<35 and test['Age'].iloc[i]>20):
marks=marks+4
feature_mix.append(marks)
 test['feature_mix']=feature_mix
test['feature_mix'].value_counts()
def model_gru_conv_3(embedding_matrix):
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = SpatialDropout1D(0.2 )(x)
 x0 = Bidirectional(CuDNNLSTM(128, kernel_initializer=initializers.glorot_uniform(seed = 2018), return_sequences=True))(x)
 x1 = Bidirectional(CuDNNGRU(64, kernel_initializer=initializers.glorot_uniform(seed = 2018), return_sequences=True))(x0)
 z = Conv1D(64, kernel_size = 1, kernel_initializer=initializers.he_uniform(seed=2018), activation = "tanh" )(x1)
 y1 = GlobalMaxPooling1D()(x1)
 y2 = GlobalMaxPooling1D()(z)
 x = concatenate([y1,y2])
 outp= Dense(1, kernel_initializer=initializers.he_uniform(seed=2018), activation='sigmoid' )(x)
 model = Model(inputs=inp, outputs=outp)
 model.compile(loss='binary_crossentropy', optimizer=Adam() , metrics=['accuracy'])
 return model<categorify>
y_test.head()
PassengerId=y_test['PassengerId']
y_test=y_test.drop(['PassengerId'],axis=1 )
SEED=2018
def model_RCNN(embedding_matrix, hidden_dim_1=128, hidden_dim_2=64,max_features=max_features):
embedding_matrix = np.concatenate([embedding_matrix,np.zeros(( 1,np.shape(embedding_matrix)[1])) ])
 print(np.shape(embedding_matrix))
left_context = Input(shape=(maxlen,))
document = Input(shape=(maxlen,))
right_context = Input(shape=(maxlen,))
embedder = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False)
 doc_embedding = embedder(document)
 doc_embedding = SpatialDropout1D(0.2 )(doc_embedding)
 l_embedding = embedder(left_context)
 l_embedding = SpatialDropout1D(0.2 )(l_embedding)
 r_embedding = embedder(right_context)
 r_embedding = SpatialDropout1D(0.2 )(r_embedding)
 forward = CuDNNLSTM(hidden_dim_1, return_sequences = True )(l_embedding)
 backward = CuDNNLSTM(hidden_dim_1, return_sequences = True )(r_embedding)
 backward = Lambda(lambda x: K.reverse(x, axes = 1))(backward)
 together = concatenate([forward, doc_embedding, backward], axis = 2)
 semantic = Conv1D(hidden_dim_2*8, kernel_size = 1, kernel_initializer=initializers.he_uniform(seed=SEED), activation = "tanh" )(together)
 pool_rnn = GlobalMaxPool1D()(semantic)
 output = Dense(1, kernel_initializer=initializers.he_uniform(seed=SEED), activation = "sigmoid" )(pool_rnn)
 model = Model(inputs = [left_context, document, right_context], outputs = output)
 model.compile(optimizer = Adam() , loss = 'binary_crossentropy', metrics = ["accuracy"])
 return model<define_search_model>
train_m=(max(train['Age'])+min(train['Age'])) /2
values={'Cabin':'nocabin','Age':train_m,'Embarked':'notknown'}
train=train.fillna(value=values)
 test_m=(max(test['Age'])+min(test['Age'])) /2
print(test_m)
 values={'Cabin':'nocabin','Age':test_m,'Embarked':'notknown',"Fare":max(test['Fare'])}
test=test.fillna(value=values)
def model_lstm_atten(embedding_matrix):
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = SpatialDropout1D(0.2 )(x)
 x0 = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)
 x2 = Bidirectional(CuDNNGRU(96, return_sequences=True))(x0)
 y2 = GlobalMaxPooling1D()(x2)
 y2 = Dropout(0.1 )(y2)
 x = Dense(1, activation="sigmoid" )(y2)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer=Adam() , metrics=['accuracy'])
 return model<choose_model_class>
y= train["Survived"]
train=train.drop(["Survived"],axis=1 )
def model_lstm_max(embedding_matrix):
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = SpatialDropout1D(0.2 )(x)
 x0 = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)
 x1 = Bidirectional(CuDNNGRU(64, kernel_initializer=initializers.glorot_uniform(seed = 2018), return_sequences=True))(x0)
 z = Conv1D(64, kernel_size = 1, kernel_initializer=initializers.he_uniform(seed=2018), activation = "tanh" )(x1)
 y2 = Attention(maxlen )(z)
 x = Dense(1, activation="sigmoid" )(y2)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer=Adam() , metrics=['accuracy'])
 return model<train_model>
X_train, X_cv, y_train, y_cv = train_test_split(train, y, stratify=y, test_size=0.2,random_state=42 )
def get_train_list(train_X):
return [np.concatenate(( np.ones(( np.shape(train_X)[0],1)) *max_features+1,train_X[:,1:]),1),train_X,np.concatenate(( np.ones(( np.shape(train_X)[0],1)) *max_features+1,train_X[:,::-1][:,1:]),1)]
def RCNN_train_pred(model, epochs=2):
train_X_list = get_train_list(train_X)
 test_X_list=get_train_list(test_X)
 val_X_list=get_train_list(val_X)
 for e in range(epochs):
model.fit(train_X_list, train_y, batch_size=512, epochs=1, validation_data=(val_X_list, val_y),callbacks=[clr])
 pred_val_y = model.predict(val_X_list, batch_size=1024, verbose=0)
 pred_test_y = model.predict(test_X_list, batch_size=1024, verbose=0)
 return pred_val_y, pred_test_y<predict_on_test>
vectorizer = CountVectorizer()
X_tr_emb =vectorizer.fit_transform(X_train['Embarked'])
 X_cv_emb =vectorizer.transform(X_cv['Embarked'])
 X_te_emb =vectorizer.transform(test['Embarked'] )
def train_pred(model, epochs=2):
for e in range(epochs):
model.fit(train_X, train_y, batch_size=512, epochs=1, validation_data=(val_X, val_y),verbose=1,callbacks=[clr])
 pred_val_y = model.predict([val_X], batch_size=1024, verbose=0)
 pred_test_y = model.predict([test_X], batch_size=1024, verbose=0)
 return pred_val_y, pred_test_y<normalization>
enc = OneHotEncoder(handle_unknown='ignore')
 X_tr_age =enc.fit_transform(np.array(X_train['Age'] ).reshape(-1,1))
X_cv_age =enc.transform(np.array(X_cv['Age'] ).reshape(-1,1))
X_te_age =enc.transform(np.array(test['Age'] ).reshape(-1,1))
train_X, val_X, test_X, train_y, val_y, word_index = load_and_prec()
max_features = len(word_index)
 print(max_features)
 embedding_matrix_1 = load_glove(word_index)
 embedding_matrix_2 = load_fasttext(word_index)
 embedding_matrix = np.concatenate([embedding_matrix_1, embedding_matrix_2], axis = 1)
 np.shape(embedding_matrix )<define_variables>
X_tr_fare =enc.fit_transform(np.array(X_train['Fare'] ).reshape(-1,1))
X_cv_fare =enc.transform(np.array(X_cv['Fare'] ).reshape(-1,1))
X_te_fare =enc.transform(np.array(test['Fare'] ).reshape(-1,1))
outputs = []<choose_model_class>
X_tr_Sbp =enc.fit_transform(np.array(X_train['SibSp'] ).reshape(-1,1))
X_cv_Sbp =enc.transform(np.array(X_cv['SibSp'] ).reshape(-1,1))
X_te_Sbp =enc.transform(np.array(test['SibSp'] ).reshape(-1,1))
clr = CyclicLR(base_lr=0.001, max_lr=0.003,step_size=300., mode='exp_range', gamma=0.99994 )<compute_train_metric>
X_tr_par =enc.fit_transform(np.array(X_train['Parch'] ).reshape(-1,1))
X_cv_par =enc.transform(np.array(X_cv['Parch'] ).reshape(-1,1))
X_te_par=enc.transform(np.array(test['Parch'] ).reshape(-1,1))
pred_val_y, pred_test_y = RCNN_train_pred(model_RCNN(embedding_matrix, hidden_dim_1=128, hidden_dim_2=64,max_features=max_features+1), epochs = 5)
 outputs.append([pred_val_y, pred_test_y, 'RCNN'])
 results = threshold_search(val_y, pred_val_y)
 print(results)
 print(confusion_matrix(val_y,pred_val_y>results['threshold']))<choose_model_class>
X_tr_pclass =enc.fit_transform(np.array(X_train['Pclass'] ).reshape(-1,1))
X_cv_pclass =enc.transform(np.array(X_cv['Pclass'] ).reshape(-1,1))
X_te_pclass =enc.transform(np.array(test['Pclass'] ).reshape(-1,1))
clr = CyclicLR(base_lr=0.001, max_lr=0.003,step_size=300., mode='exp_range', gamma=0.99994 )<compute_train_metric>
X_tr_sex =vectorizer.fit_transform(X_train['Sex'])
 X_cv_sex =vectorizer.transform(X_cv['Sex'])
 X_te_sex =vectorizer.transform(test['Sex'] )
pred_val_y, pred_test_y = train_pred(model_lstm_atten(embedding_matrix), epochs = 4)
 outputs.append([pred_val_y, pred_test_y, 'LSTM w/ max'])
 results = threshold_search(val_y, pred_val_y)
 print(results)
 print(confusion_matrix(val_y,pred_val_y>results['threshold']))<choose_model_class>
X_tr_cabin =vectorizer.fit_transform(X_train['Cabin'])
 X_cv_cabin =vectorizer.transform(X_cv['Cabin'])
 X_te_cabin =vectorizer.transform(test['Cabin'] )
clr = CyclicLR(base_lr=0.001, max_lr=0.003,step_size=300., mode='exp_range', gamma=0.99994 )<compute_train_metric>
X_tr_tkt =vectorizer.fit_transform(X_train['Ticket'])
 X_cv_tkt =vectorizer.transform(X_cv['Ticket'])
 X_te_tkt =vectorizer.transform(test['Ticket'] )
pred_val_y, pred_test_y = train_pred(model_gru_conv_3(embedding_matrix), epochs = 4)
 outputs.append([pred_val_y, pred_test_y, 'LSTM conv 3'])
 results = threshold_search(val_y, pred_val_y)
 print(results)
 print(confusion_matrix(val_y,pred_val_y>results['threshold']))<choose_model_class>
X_tr_fmix =enc.fit_transform(np.array(X_train['feature_mix'] ).reshape(-1,1))
X_cv_fmix =enc.transform(np.array(X_cv['feature_mix'] ).reshape(-1,1))
X_te_fmix =enc.transform(np.array(test['feature_mix'] ).reshape(-1,1))
clr = CyclicLR(base_lr=0.001, max_lr=0.003,step_size=300., mode='exp_range', gamma=0.99994 )<compute_train_metric>

 X_tr = hstack(( X_tr_age,X_tr_fare,X_tr_sex,X_tr_pclass,X_tr_emb)).tocsr()
X_cv = hstack(( X_cv_age,X_cv_fare,X_cv_sex,X_cv_pclass,X_cv_emb)).tocsr()
X_te = hstack(( X_te_age,X_te_fare,X_te_sex,X_te_pclass,X_te_emb)).tocsr()
print(X_tr.shape)
 print(X_te.shape)
 print(X_cv.shape )
pred_val_y, pred_test_y = train_pred(model_lstm_max(embedding_matrix), epochs = 4)
 outputs.append([pred_val_y, pred_test_y, 'LSTM w/ atten'])
 results = threshold_search(val_y, pred_val_y)
 print(results)
 print(confusion_matrix(val_y,pred_val_y>results['threshold']))<compute_train_metric>
alpha = [10 ** x for x in range(-5, 1)]
cv_log_error_array=[]
for i in alpha:
clf = SGDClassifier(alpha=i,class_weight="balanced", penalty='l2', loss='log', random_state=42)
 clf.fit(X_tr, y_train)
 sig_clf = CalibratedClassifierCV(clf, method="sigmoid")
 sig_clf.fit(X_tr, y_train)
 predict_y = sig_clf.predict_proba(X_cv)
 cv_log_error_array.append(log_loss(y_cv, predict_y, labels=clf.classes_, eps=1e-15))
print('For values of alpha = ', i, "The log loss is:",log_loss(y_cv, predict_y, labels=clf.classes_, eps=1e-15))
fig, ax = plt.subplots()
ax.plot(alpha, cv_log_error_array,c='g')
 for i, txt in enumerate(np.round(cv_log_error_array,3)) :
ax.annotate(( alpha[i],np.round(txt,3)) ,(alpha[i],cv_log_error_array[i]))
plt.grid()
plt.title("Cross Validation Error for each alpha")
 plt.xlabel("Alpha i's")
 plt.ylabel("Error measure")
 plt.show()
best_alpha = np.argmin(cv_log_error_array)
 clf = SGDClassifier(alpha=alpha[best_alpha],class_weight="balanced", penalty='l2', loss='log', random_state=42)
 clf.fit(X_tr, y_train)
 sig_clf = CalibratedClassifierCV(clf, method="sigmoid")
 sig_clf.fit(X_tr, y_train)
 predict_y = sig_clf.predict_proba(X_tr)
 print('For values of best alpha = ', alpha[best_alpha], "The train log loss is:",log_loss(y_train, predict_y, labels=clf.classes_, eps=1e-15))
predict_y = sig_clf.predict_proba(X_cv)
 print('For values of best alpha = ', alpha[best_alpha], "The cross validation log loss is:",log_loss(y_cv, predict_y, labels=clf.classes_, eps=1e-15))
predict_y = sig_clf.predict_proba(X_te)
 print('For values of best alpha = ', alpha[best_alpha], "The test log loss is:",log_loss(y_test, predict_y, labels=clf.classes_, eps=1e-15))
lr=alpha[best_alpha]
coefs = [0.35,0.25,0.2,0.2]
pred_val_y = np.sum([outputs[i][0]*coefs[i] for i in range(len(outputs)) ], axis = 0)
 results = threshold_search(val_y, pred_val_y)
 print(results)
 print(confusion_matrix(val_y,pred_val_y>results['threshold']))<save_to_csv>
pred=sig_clf.predict(X_te)
 df=pd.DataFrame(zip(PassengerId,pred),columns=['PassengerId',"Survived"])
 df
df.to_csv('/kaggle/working/output.csv',index=False )
<import_modules><EOS> 
d=pd.read_csv('/kaggle/working/output.csv')
 d