Kaggle competition: Titanic - Machine Learning from Disaster
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim
def f1(y_true, y_pred):
def recall(y_true, y_pred):
true_positives = K.sum(K.round(K.clip(y_true*y_pred, 0, 1)))
 possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
 recall = true_positives/(possible_positives + K.epsilon())
 return recall
def precision(y_true, y_pred):
true_positives = K.sum(K.round(K.clip(y_true*y_pred, 0, 1)))
 predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
 precision = true_positives/(predicted_positives + K.epsilon())
 return precision
precision = precision(y_true, y_pred)
 recall = recall(y_true, y_pred)
 return 2*(( precision*recall)/(precision+recall+K.epsilon()))<compute_test_metric>
%matplotlib inline
warnings.filterwarnings('ignore')
def f1(y_true, y_pred):
def recall(y_true, y_pred):
true_positives = K.sum(K.round(K.clip(y_true*y_pred, 0, 1)))
 possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
 recall = true_positives/(possible_positives + K.epsilon())
 return recall
def precision(y_true, y_pred):
true_positives = K.sum(K.round(K.clip(y_true*y_pred, 0, 1)))
 predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
 precision = true_positives/(predicted_positives + K.epsilon())
 return precision
precision = precision(y_true, y_pred)
 recall = recall(y_true, y_pred)
 return 2*(( precision*recall)/(precision+recall+K.epsilon()))<choose_model_class>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 all_data = [train,test]
def model_lstm_att(embedding_matrix):
inp = Input(shape=(MAXLEN,))
x = Embedding(MAX_FEATURES, EMBED_SIZE, weights=[embedding_matrix], trainable=False )(inp)
 x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(32, return_sequences=True))(x)
 att = Attention(MAXLEN )(x)
 y = Dense(32, activation='relu' )(att)
 y = Dropout(0.1 )(y)
 outp = Dense(1, activation='sigmoid' )(y)
 model = Model(inputs=inp, outputs=outp)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['binary_accuracy', 'accuracy'])
 return model<load_pretrained>
cor_map(train.drop(['PassengerId'],axis=1))
paragram = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
embedding_matrix_para = load_glove_matrix(tokenizer.word_index, load_embed(paragram))<compute_test_metric>
train[['Pclass','Survived']].groupby(['Pclass'], as_index=False ).mean().sort_values(by='Survived',ascending=False )
embedding_matrix = np.mean([embedding_matrix_glove, embedding_matrix_para], axis=0 )<find_best_params>
train[['Sex','Survived']].groupby(['Sex'],as_index=False ).mean().sort_values(by='Survived',ascending=False )
def train_pred(model, epochs=2):
for e in range(epochs):
model.fit(xtrain, ytrain, batch_size=512, epochs=3, validation_data=(xval, yval))
pred_val_y = model.predict([xval], batch_size=1024, verbose=1)
 best_thresh = 0.5
best_score = 0.0
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 score = metrics.f1_score(yval,(pred_val_y > thresh ).astype(int))
if score > best_score:
best_thresh = thresh
best_score = score
print("Val puntuacion F1: {:.4f}".format(best_score))
pred_test_y = model.predict([xtest], batch_size=1024, verbose=1)
 return pred_val_y, pred_test_y, best_score<predict_on_test>
train[['Embarked','Survived']].groupby(['Embarked'],as_index=False ).mean().sort_values(by='Survived',ascending=False )
outputs = []
pred_val_y, pred_test_y, best_score = train_pred(model_lstm, epochs=2)
 outputs.append([pred_val_y, pred_test_y, best_score, 'model_lstm_att only Glove'] )<compute_test_metric>
guess_ages = np.zeros(( 3,9))
for dataset in all_data:
dataset['ageFill']=dataset.Age.isnull().map({False:0,True:1})
 med_all = dataset['Age'].median()
for i in range(0,3):
for j in range(0,9):
guess_df=dataset[(dataset['Pclass']==i+1)&\ 
(dataset['SibSp']==j)]['Age'].dropna()
age_guess=guess_df.median()
try:
guess_ages[i,j] = int(age_guess/0.5+0.5)*0.5
except:
guess_ages[i,j]=med_all
for i in range(0,3):
for j in range(0,9):
dataset.loc[(dataset.Age.isnull())&(dataset.Pclass==i+1)&(dataset.SibSp==j),'Age']=guess_ages[i,j]
dataset['Age']=dataset['Age'].astype(int)
 print(guess_ages)
 train.head(10 )
outputs.sort(key=lambda x: x[2])
 weights = [i for i in range(1, len(outputs)+ 1)]
weights = [float(i)/ sum(weights)for i in weights]
pred_val_y = np.mean([outputs[i][0] for i in range(len(outputs)) ], axis = 0)
 thresholds = []
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 res = metrics.f1_score(yval,(pred_val_y > thresh ).astype(int))
thresholds.append([thresh, res])
 print("La puntuación F1 en los limites {0} y {1}".format(thresh, res))
thresholds.sort(key=lambda x: x[1], reverse=True)
 best_thresh = thresholds[0][0]<train_model>
freq_port = train.Embarked.dropna().mode() [0]
train.Embarked = train.Embarked.fillna(freq_port)
 print(freq_port)
print("Mejor limite:", best_thresh, "y puntuacion F1 ", thresholds[0][1] )<data_type_conversions>
test['Fare']=test.Fare.fillna(test.Fare.mean())
 test.info()
pred_test_y = np.mean([outputs[i][1] for i in range(len(outputs)) ], axis = 0)
 pred_test_y =(pred_test_y > best_thresh ).astype(int )<save_to_csv>
for dataset in all_data:
dataset['Title'] = dataset.Name.str.extract('([A-Za-z]+)\.',expand=False)
 pd.crosstab(train['Title'],train['Sex'] )
sub = pd.read_csv('.. /input/sample_submission.csv')
 out_df = pd.DataFrame({"qid":sub["qid"].values})
 out_df['prediction'] = pred_test_y
out_df.to_csv("submission.csv", index=False )<import_modules>
def cleanTicket(ticket):
ticket = ticket.replace('.' , '')
 ticket = ticket.replace('/' , '')
 ticket = ticket.split()
ticket = map(lambda t : t.strip() , ticket)
 ticket = list(filter(lambda t : not t.isdigit() , ticket))
if len(ticket)> 0:
return ticket[0]
else:
return 'XXX'
for dataset in all_data:
dataset[ 'ticketPos' ] = dataset[ 'Ticket' ].map(cleanTicket)
 train.head()
ps = PorterStemmer()
lc = LancasterStemmer()
sb = SnowballStemmer("english")
<set_options>
for dataset in all_data:
title_mapping = {'Mr':1,'Rare':2,'Master':3,'Miss':4,'Mrs':5}
dataset['Title'] = dataset['Title'].map(title_mapping)
 dataset['Title'] = dataset['Title'].fillna(0)
 train.head()
gc.collect()
K.clear_session()<normalization>
for dataset in all_data:
dataset['Sex']=dataset.Sex.map({'male':0,'female':1})
 train.head()
class AttentionWeightedAverage(Layer):
def __init__(self, return_attention=False, **kwargs):
self.init = initializers.get('uniform')
 self.supports_masking = True
self.return_attention = return_attention
super(AttentionWeightedAverage, self ).__init__(** kwargs)
 def build(self, input_shape):
self.input_spec = [InputSpec(ndim=3)]
assert len(input_shape)== 3
self.W = self.add_weight(shape=(input_shape[2], 1),
name='{}_W'.format(self.name),
initializer=self.init)
 self.trainable_weights = [self.W]
super(AttentionWeightedAverage, self ).build(input_shape)
 def call(self, x, mask=None):
logits = K.dot(x, self.W)
 x_shape = K.shape(x)
 logits = K.reshape(logits,(x_shape[0], x_shape[1]))
ai = K.exp(logits - K.max(logits, axis=-1, keepdims=True))
if mask is not None:
mask = K.cast(mask, K.floatx())
 ai = ai * mask
att_weights = ai /(K.sum(ai, axis=1, keepdims=True)+ K.epsilon())
 weighted_input = x * K.expand_dims(att_weights)
 result = K.sum(weighted_input, axis=1)
 if self.return_attention:
return [result, att_weights]
return result
def get_output_shape_for(self, input_shape):
return self.compute_output_shape(input_shape)
 def compute_output_shape(self, input_shape):
output_len = input_shape[2]
if self.return_attention:
return [(input_shape[0], output_len),(input_shape[0], input_shape[1])]
return(input_shape[0], output_len)
 def compute_mask(self, input, input_mask=None):
if isinstance(input_mask, list):
return [None] * len(input_mask)
 else:
return None<load_pretrained>
for dataset in all_data:
dataset['Embarked']=dataset.Embarked.map({'S':0,'Q':1,"C":2})
 train.head()
spell_model = gensim.models.KeyedVectors.load_word2vec_format('.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec')
 words = spell_model.index2word
w_rank = {}
for i,word in enumerate(words):
w_rank[word] = i
WORDS = w_rank<set_options>
for dataset in all_data:
dataset['AgeBand']=pd.cut(dataset['Age'],5,labels=[0,1,2,3,4])
 dataset['AgeBand']=dataset.AgeBand.astype(int)
 train[['AgeBand','Survived']].groupby(['AgeBand'],as_index=False ).mean().sort_values(by='Survived',ascending=False )
del spell_model, w_rank
gc.collect()<categorify>
for dataset in all_data:
dataset['cabinRec']=dataset.Cabin.isnull().map({False:0,True:1})
 train.head()
def words(text): return re.findall(r'\w+', text.lower())
 def P(word):
"Probability of `word`."
return - WORDS.get(word, 0)
 def correction(word):
"Most probable spelling correction for word."
return max(candidates(word), key=P)
 def candidates(word):
"Generate possible spelling corrections for word."
return(known([word])or known(edits1(word)) or [word])
 def known(words):
"The subset of `words` that appear in the dictionary of WORDS."
return set(w for w in words if w in WORDS)
 def edits1(word):
"All edits that are one edit away from `word`."
letters = 'abcdefghijklmnopqrstuvwxyz'
splits = [(word[:i], word[i:])for i in range(len(word)+ 1)]
deletes = [L + R[1:] for L, R in splits if R]
transposes = [L + R[1] + R[0] + R[2:] for L, R in splits if len(R)>1]
replaces = [L + c + R[1:] for L, R in splits if R for c in letters]
inserts = [L + c + R for L, R in splits for c in letters]
return set(deletes + transposes + replaces + inserts)
 def edits2(word):
"All edits that are two edits away from `word`."
return(e2 for e1 in edits1(word)for e2 in edits1(e1))
def singlify(word):
return "".join([letter for i,letter in enumerate(word)if i == 0 or letter != word[i-1]] )<features_selection>
for dataset in all_data:
dataset['Fare'].fillna(dataset['Fare'].median() , inplace = True)
 dataset['FareBin'] = pd.qcut(dataset['Fare'], 5,labels=[1,2,3,4,5])
 dataset['FareBin'] = dataset.FareBin.astype(int)
 train.head()
def load_glove(word_dict, lemma_dict):
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
embed_size = 300
nb_words = len(word_dict)+1
embedding_matrix = np.zeros(( nb_words, embed_size), dtype=np.float32)
 unknown_vector = np.zeros(( embed_size,), dtype=np.float32)- 1.
 print(unknown_vector[:5])
 for key in tqdm(word_dict):
word = key
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = key.lower()
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = key.upper()
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = key.capitalize()
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = ps.stem(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = lc.stem(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = sb.stem(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = lemma_dict[key]
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
if len(key)> 1:
word = correction(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
embedding_matrix[word_dict[key]] = unknown_vector
return embedding_matrix, nb_words<split>
def load_fasttext(word_dict, lemma_dict):
EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100)
 embed_size = 300
nb_words = len(word_dict)+1
embedding_matrix = np.zeros(( nb_words, embed_size), dtype=np.float32)
 unknown_vector = np.zeros(( embed_size,), dtype=np.float32)- 1.
 print(unknown_vector[:5])
 for key in tqdm(word_dict):
word = key
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = key.lower()
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = key.upper()
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = key.capitalize()
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = ps.stem(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = lc.stem(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = sb.stem(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = lemma_dict[key]
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
if len(key)> 1:
word = correction(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
embedding_matrix[word_dict[key]] = unknown_vector
return embedding_matrix, nb_words<split>
train_df=train.drop(['AgeBand','FareBin','Ticket','Cabin','Name','PassengerId','ticketPos'],axis=1)
 test_df=test.drop(['AgeBand','FareBin','Ticket','Cabin','Name','PassengerId','ticketPos'],axis=1)
 combine_df = [train_df,test_df]
train_df.head()
def load_para(word_dict, lemma_dict):
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 embed_size = 300
nb_words = len(word_dict)+1
embedding_matrix = np.zeros(( nb_words, embed_size), dtype=np.float32)
 unknown_vector = np.zeros(( embed_size,), dtype=np.float32)- 1.
 print(unknown_vector[:5])
 for key in tqdm(word_dict):
word = key
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = key.lower()
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = key.upper()
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = key.capitalize()
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = ps.stem(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = lc.stem(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = sb.stem(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = lemma_dict[key]
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
if len(key)> 1:
word = correction(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
embedding_matrix[word_dict[key]] = unknown_vector
return embedding_matrix, nb_words<choose_model_class>
for dataset in combine_df:
dataset['Cabin_Class'] =(dataset.cabinRec+1)*dataset.Pclass
def build_model(embedding_matrix, nb_words, embedding_size=300):
inp = Input(shape=(max_length,))
x = Embedding(nb_words, embedding_size, weights=[embedding_matrix], trainable=False )(inp)
 x = SpatialDropout1D(0.3 )(x)
 x1 = Bidirectional(LSTM(256, return_sequences=True))(x)
 x2 = Bidirectional(GRU(128, return_sequences=True))(x1)
 max_pool1 = GlobalMaxPooling1D()(x1)
 max_pool2 = GlobalMaxPooling1D()(x2)
 conc = Concatenate()([max_pool1, max_pool2])
 predictions = Dense(1, activation='sigmoid' )(conc)
 model = Model(inputs=inp, outputs=predictions)
 adam = optimizers.Adam(lr=learning_rate)
 model.compile(optimizer=adam, loss='binary_crossentropy', metrics=['accuracy'])
 return model<load_from_csv>
train_df=train_df.drop('Sex',axis=1)
 test_df=test_df.drop('Sex',axis=1 )
start_time = time.time()
print("Loading data...")
 train = pd.read_csv(".. /input/train.csv" ).fillna(' ')
 test = pd.read_csv('.. /input/test.csv' ).fillna(' ')
 train_text = train['question_text']
test_text = test['question_text']
text_list = pd.concat([train_text, test_text])
 y = train['target'].values
num_train_data = y.shape[0]
print("--- %s seconds ---" %(time.time() - start_time))
start_time = time.time()
print("Spacy NLP...")
 nlp = spacy.load('en_core_web_lg', disable=['parser','ner','tagger'])
 nlp.vocab.add_flag(lambda s: s.lower() in spacy.lang.en.stop_words.STOP_WORDS, spacy.attrs.IS_STOP)
 word_dict = {}
word_index = 1
lemma_dict = {}
docs = nlp.pipe(text_list, n_threads = 2)
 word_sequences = []
for doc in tqdm(docs):
word_seq = []
for token in doc:
if(token.text not in word_dict)and(token.pos_ is not "PUNCT"):
word_dict[token.text] = word_index
word_index += 1
lemma_dict[token.text] = token.lemma_
if token.pos_ is not "PUNCT":
word_seq.append(word_dict[token.text])
 word_sequences.append(word_seq)
 del docs
gc.collect()
train_word_sequences = word_sequences[:num_train_data]
test_word_sequences = word_sequences[num_train_data:]
del word_sequences
gc.collect()
print("--- %s seconds ---" %(time.time() - start_time))
max_length = 55
embedding_size = 600
learning_rate = 0.001
batch_size = 512
num_epoch = 4
train_word_sequences = pad_sequences(train_word_sequences, maxlen=max_length, padding='post')
 test_word_sequences = pad_sequences(test_word_sequences, maxlen=max_length, padding='post')
 pred_prob = np.zeros(( len(test_word_sequences),), dtype=np.float32)
 start_time = time.time()
print("Loading embedding matrix...")
 embedding_matrix_glove, nb_words = load_glove(word_dict, lemma_dict)
 embedding_matrix_fasttext, nb_words = load_fasttext(word_dict, lemma_dict)
 embedding_matrix = np.concatenate(( embedding_matrix_glove, embedding_matrix_fasttext), axis=1)
 print("--- %s seconds ---" %(time.time() - start_time))<train_model>
X_train_valid = train_df.drop('Survived',axis=1)
 y_train_valid = train_df['Survived']
X_test = test_df
X_train,X_valid,y_train,y_valid=train_test_split(X_train_valid,y_train_valid,test_size=0.25,random_state=0)
 print(X_train.shape,X_test.shape,X_valid.shape )
print("Start training...")
 start_time = time.time()
model = build_model(embedding_matrix, nb_words, embedding_size)
 model.summary()<train_model>
gradb= GradientBoostingClassifier(learning_rate=0.01,random_state=0,n_estimators=2000,max_features=4)
 gradb.fit(X_train,y_train)
 print(gradb.score(X_valid,y_valid))
plot_model_var_imp(gradb,X_train,y_train )
model.fit(train_word_sequences, y, batch_size=batch_size, epochs=num_epoch-1, verbose=2)
 pred_prob += 0.15*np.squeeze(model.predict(test_word_sequences, batch_size=batch_size, verbose=2))
model.fit(train_word_sequences, y, batch_size=batch_size, epochs=1, verbose=2)
 pred_prob += 0.35*np.squeeze(model.predict(test_word_sequences, batch_size=batch_size, verbose=2))
del model, embedding_matrix_fasttext, embedding_matrix
gc.collect()
K.clear_session()
print("--- %s seconds ---" %(time.time() - start_time))<train_model>
clf = XGBClassifier(random_state=0,n_jobs=-1)
 cv_sets = ShuffleSplit(X_train.shape[0], n_iter =5, test_size = 0.20, random_state = 7)
 parameters = {'n_estimators':list(range(100,1000,100)) ,
'learning_rate':[0.05,0.1,0.25,0.5,0.75],
'reg_lambda':[1,10,15,20,25]}
acc_scorer=make_scorer(accuracy_score)
 grid_obj=GridSearchCV(clf, parameters, scoring=acc_scorer,verbose=1,cv=cv_sets)
 grid_obj= grid_obj.fit(X_train,y_train)
 clf_best = grid_obj.best_estimator_
clf_best.fit(X_train,y_train )
<train_model><EOS> 
ids=test['PassengerId']
predictions = clf_best.predict(X_test)
 my_submission = pd.DataFrame({ 'PassengerId' : ids, 'Survived': predictions })
 my_submission.to_csv('submission.csv', index=False)