Kaggle competition: Titanic - Machine Learning from Disaster
date_list = []
date = pd.date_range(date_max_train+timedelta(days=1), date_max_test)
 results = pd.DataFrame({'date': date, 'preds':preds})
 results.head()<merge>
training_df = pd.read_csv(".. /input/train.csv")
 testing_df = pd.read_csv(".. /input/test.csv")
 combine = [training_df, testing_df]
test_merged = test.merge(train[['province_state', 'country_region', 'date', 'confirmedcases', 'fatalities']],
left_on=['province_state', 'country_region', 'date'],
right_on=['province_state', 'country_region', 'date'],
how='left')
 test_merged<merge>
training_df[['Pclass', 'Survived']].groupby(['Pclass'], as_index=False ).mean().sort_values(by='Survived', ascending=False )
test_merged2 = test_merged.merge(results,
left_on=['province_state', 'country_region', 'date'],
right_on=['province_state', 'country_region', 'date'],
how='left')
 test_merged2<drop_column>
training_df[['Sex', 'Survived']].groupby(['Sex'], as_index=False ).mean().sort_values(by='Survived', ascending=False )
test_merged2['confirmedcases'] = test_merged2.apply(lambda row: row['confirmedcases_x'] if pd.isnull(row['confirmedcases_y'])else row['confirmedcases_y'], axis=1)
 test_merged2.drop(['confirmedcases_x', 'confirmedcases_y'], axis=1, inplace=True)
 test_merged2<merge>
training_df[['SibSp', 'Survived']].groupby(['SibSp'], as_index=False ).mean().sort_values(by='Survived', ascending=False )
test_merged3 = test_merged2.merge(results,
left_on=['province_state', 'country_region', 'date'],
right_on=['province_state', 'country_region', 'date'],
how='left')
 test_merged3<drop_column>
training_df[['Parch', 'Survived']].groupby(['Parch'], as_index=False ).mean().sort_values(by='Survived', ascending=False )
test_merged3['fatalities'] = test_merged3.apply(lambda row: row['fatalities_x'] if pd.isnull(row['fatalities_y'])else row['fatalities_y'], axis=1)
 test_merged3.drop(['fatalities_x', 'fatalities_y'], axis=1, inplace=True)
 test_merged3<drop_column>
column_choice_training.Sex[column_choice_training.Sex == 'female'] = 0
column_choice_training.Sex[column_choice_training.Sex == 'male'] = 1
column_choice_test.Sex[column_choice_test.Sex == 'female'] = 0
column_choice_test.Sex[column_choice_test.Sex == 'male'] = 1
column_choice_training.head()
submission = test_merged3.copy()
submission.drop(['country_region', 'province_state', 'date'], axis=1, inplace=True)
 submission.rename(columns={'forecastid': 'ForecastId',
'fatalities': 'Fatalities',
'confirmedcases': 'ConfirmedCases'}, inplace=True)
 submission<save_to_csv>
freq_port = column_choice_training.Embarked.dropna().mode() [0]
column_choice_training.Embarked = column_choice_training.Embarked.fillna(freq_port)
 column_choice_test.Embarked = column_choice_test.Embarked.fillna(freq_port)
 column_choice_training.Embarked[column_choice_training.Embarked == 'S'] = 0
column_choice_training.Embarked[column_choice_training.Embarked == 'C'] = 1
column_choice_training.Embarked[column_choice_training.Embarked == 'Q'] = 2
column_choice_test.Embarked[column_choice_test.Embarked == 'S'] = 0
column_choice_test.Embarked[column_choice_test.Embarked == 'C'] = 1
column_choice_test.Embarked[column_choice_test.Embarked == 'Q'] = 2
column_choice_training.head()
submission.to_csv("submission.csv", index=False )<load_from_csv>
column_choice_training['Age_is_Null'] = column_choice_training['Age'].apply(lambda x: 1 if pd.isnull(x)else 0)
 column_choice_test['Age_is_Null'] = column_choice_test['Age'].apply(lambda x: 1 if pd.isnull(x)else 0 )
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 print("Train shape : ",train_df.shape)
 print("Test shape : ",test_df.shape )<split>
column_choice_training[['Age', 'Pclass', 'Sex']].groupby(['Pclass', 'Sex'], as_index=False ).mean()
train_df, val_df = train_test_split(train_df, test_size=0.1, random_state=2018)
 embed_size = 300
max_features = 50000
maxlen = 100
train_X = train_df["question_text"].fillna("_na_" ).values
val_X = val_df["question_text"].fillna("_na_" ).values
test_X = test_df["question_text"].fillna("_na_" ).values
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 val_X = tokenizer.texts_to_sequences(val_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 train_X = pad_sequences(train_X, maxlen=maxlen)
 val_X = pad_sequences(val_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 train_y = train_df['target'].values
val_y = val_df['target'].values<prepare_x_and_y>
age_means = column_choice_training.groupby(['Sex', 'Pclass'])['Age']
column_choice_training.Age = age_means.transform(lambda x: x.fillna(x.mean()))
column_choice_test.Age = age_means.transform(lambda x: x.fillna(x.mean()))
column_choice_training.head(6 )
np.random.seed(2018)
 trn_idx = np.random.permutation(len(train_X))
val_idx = np.random.permutation(len(val_X))
train_X = train_X[trn_idx]
val_X = val_X[val_idx]
train_y = train_y[trn_idx]
val_y = val_y[val_idx]<import_modules>
bins = [0, 6, 60, 80]
column_choice_training['Age_cut'] = pd.cut(column_choice_training['Age'], bins)
 column_choice_test['Age_cut'] = pd.cut(column_choice_test['Age'], bins)
 column_choice_training = pd.concat([column_choice_training.drop(['Age_cut'], axis=1),
pd.get_dummies(column_choice_training['Age_cut'], prefix = 'Age_cut')], axis=1)
 column_choice_training = column_choice_training.drop(['Age_cut_(6, 60]'], axis=1)
 column_choice_test = pd.concat([column_choice_test.drop(['Age_cut'], axis=1),
pd.get_dummies(column_choice_test['Age_cut'], prefix = 'Age_cut')], axis=1)
 column_choice_test = column_choice_test.drop(['Age_cut_(6, 60]'], axis=1)
 column_choice_training.head(6 )
from keras.models import Sequential,Model
from keras.layers import Dense, CuDNNLSTM, Bidirectional, Input, Dropout, Embedding, CuDNNGRU, GlobalMaxPool1D
from keras.optimizers import Adam
from keras import backend as K
from keras.engine.topology import Layer
from keras import initializers, regularizers, constraints<statistical_test>
column_choice_training.Fare = column_choice_training.Fare.fillna(column_choice_training.Fare.mean())
 column_choice_test.Fare = column_choice_test.Fare.fillna(column_choice_training.Fare.mean())
 column_choice_training.head(6 )
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector<choose_model_class>
column_choice_training['Name_Length'] = column_choice_training['Name'].apply(len)
 column_choice_test['Name_Length'] = column_choice_test['Name'].apply(len)
 column_choice_training.head(6 )
filter_sizes = [1,2,3,5]
num_filters = 36
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Reshape(( maxlen, embed_size, 1))(x)
 maxpool_pool = []
for i in range(len(filter_sizes)) :
conv = Conv2D(num_filters, kernel_size=(filter_sizes[i], embed_size),
kernel_initializer='he_normal', activation='elu' )(x)
 maxpool_pool.append(MaxPool2D(pool_size=(maxlen - filter_sizes[i] + 1, 1))(conv))
z = Concatenate(axis=1 )(maxpool_pool)
 z = Flatten()(z)
 z = Dropout(0.1 )(z)
 outp = Dense(1, activation="sigmoid" )(z)
 model = Model(inputs=inp, outputs=outp)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'] )<train_model>
column_choice_training['Title'] = column_choice_training['Name'].str.split(', ' ).str[1]
column_choice_training['Title'] = column_choice_training['Title'].str.split('.' ).str[0]
column_choice_training = column_choice_training.drop(['Name'], axis=1)
 column_choice_test['Title'] = column_choice_test['Name'].str.split(', ' ).str[1]
column_choice_test['Title'] = column_choice_test['Title'].str.split('.' ).str[0]
column_choice_test = column_choice_test.drop(['Name'], axis=1 )
model.fit(train_X, train_y, batch_size=512, epochs=2, validation_data=(val_X, val_y))<predict_on_test>
column_choice_training['Title'] = column_choice_training['Title'].replace(['the Countess','Capt', 'Col', 'Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer'], 'Rare')
 column_choice_training['Title'] = column_choice_training['Title'].replace(['Lady','Mlle', 'Ms'], 'Mrs')
 column_choice_training['Title'] = column_choice_training['Title'].replace(['Mme'], 'Miss')
 column_choice_test['Title'] = column_choice_test['Title'].replace(['Col', 'Dona', 'Dr', 'Rev'], 'Rare')
 column_choice_test['Title'] = column_choice_test['Title'].replace(['Ms'], 'Mrs')
 pd.crosstab(column_choice_training['Title'], column_choice_training['Sex'] )
pred_cnn_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_cnn_val_y>thresh ).astype(int))))<predict_on_test>
column_choice_training.Title[column_choice_training.Title == 'Master'] = 0
column_choice_training.Title[column_choice_training.Title == 'Miss'] = 1
column_choice_training.Title[column_choice_training.Title == 'Mr'] = 2
column_choice_training.Title[column_choice_training.Title == 'Mrs'] = 3
column_choice_training.Title[column_choice_training.Title == 'Rare'] = 4
column_choice_test.Title[column_choice_test.Title == 'Master'] = 0
column_choice_test.Title[column_choice_test.Title == 'Miss'] = 1
column_choice_test.Title[column_choice_test.Title == 'Mr'] = 2
column_choice_test.Title[column_choice_test.Title == 'Mrs'] = 3
column_choice_test.Title[column_choice_test.Title == 'Rare'] = 4
column_choice_training.head(6 )
pred_cnn_test_y = model.predict([test_X], batch_size=1024, verbose=1 )<set_options>
column_choice_training['Ticket_Len'] = column_choice_training['Ticket'].apply(len)
 column_choice_test['Ticket_Len'] = column_choice_test['Ticket'].apply(len)
 print(column_choice_test['Ticket_Len'].value_counts())
 column_choice_training[['Ticket_Len', 'Survived']].groupby(['Ticket_Len'], as_index=False ).mean().sort_values(by='Survived', ascending=False )
del word_index, embeddings_index, all_embs, embedding_matrix, model, inp, x
time.sleep(10 )<choose_model_class>
column_choice_training['Ticket_Letter'] = column_choice_training['Ticket'].str[0]
column_choice_test['Ticket_Letter'] = column_choice_test['Ticket'].str[0]
print(column_choice_training['Ticket_Letter'].value_counts())
 print(column_choice_test['Ticket_Letter'].value_counts())
 column_choice_training[['Ticket_Letter', 'Survived']].groupby(['Ticket_Letter'], as_index=False ).mean().sort_values(by='Survived', ascending=False )
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim<statistical_test>
column_choice_training['Ticket_Letter'] = column_choice_training['Ticket_Letter'].replace(['W', '4', '7', '6', 'L', '5', '8'], 'Rare_Low_Surv')
 column_choice_training['Ticket_Letter'] = column_choice_training['Ticket_Letter'].replace(['F', '9'], 'Rare_High_Surv')
 column_choice_test['Ticket_Letter'] = column_choice_test['Ticket_Letter'].replace(['W', '4', '7', '6', 'L', '5', '8'], 'Rare_Low_Surv')
 column_choice_test['Ticket_Letter'] = column_choice_test['Ticket_Letter'].replace(['F', '9'], 'Rare_High_Surv')
 column_choice_training.tail()
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)
 x = Attention(maxlen )(x)
 x = Dense(64, activation="relu" )(x)
 x = Dropout(0.1 )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer=Adam(lr=1e-3), metrics=['accuracy'])
 print(model.summary() )<train_model>
column_choice_training = pd.concat([column_choice_training.drop(['Ticket', 'Ticket_Letter'], axis=1),
pd.get_dummies(column_choice_training['Ticket_Letter'], prefix = 'Ticket_Letter')], axis=1)
 column_choice_test = pd.concat([column_choice_test.drop(['Ticket', 'Ticket_Letter'], axis=1),
pd.get_dummies(column_choice_test['Ticket_Letter'], prefix = 'Ticket_Letter')], axis=1)
 column_choice_training.head()
model.fit(train_X, train_y, batch_size=512, epochs=2, validation_data=(val_X, val_y))<predict_on_test>
column_choice_training.Cabin = column_choice_training.Cabin.fillna('N0')
 column_choice_test.Cabin = column_choice_test.Cabin.fillna('N0')
 column_choice_training['Cabin_Letter'] = column_choice_training['Cabin'].str[0]
column_choice_test['Cabin_Letter'] = column_choice_test['Cabin'].str[0]
print(column_choice_training['Cabin_Letter'].value_counts())
 print(column_choice_test['Cabin_Letter'].value_counts())
 column_choice_training[['Cabin_Letter', 'Survived']].groupby(['Cabin_Letter'], as_index=False ).mean().sort_values(by='Survived', ascending=False )
pred_glove_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_glove_val_y>thresh ).astype(int))))<predict_on_test>
column_choice_training['Cabin_Number'] = column_choice_training['Cabin'].str.split(' ' ).str[-1].str[1:]
column_choice_test['Cabin_Number'] = column_choice_test['Cabin'].str.split(' ' ).str[-1].str[1:]
column_choice_training['Cabin_Number'] = column_choice_training['Cabin_Number'].replace(['0'], np.NaN)
 column_choice_test['Cabin_Number'] = column_choice_test['Cabin_Number'].replace(['0'], np.NaN)
 column_choice_training['Cabin_Number'] = column_choice_training['Cabin_Number'].apply(lambda x: int(x)if not pd.isnull(x)and x != '' else np.NaN)
 column_choice_test['Cabin_Number'] = column_choice_test['Cabin_Number'].apply(lambda x: int(x)if not pd.isnull(x)and x != '' else np.NaN)
 column_choice_training['Cabin_Number_cut'] = pd.qcut(column_choice_training['Cabin_Number'],4)
 column_choice_test['Cabin_Number_cut'] = pd.qcut(column_choice_test['Cabin_Number'],4)
 column_choice_training = pd.concat([column_choice_training.drop(['Cabin_Number', 'Cabin_Number_cut'], axis=1),
pd.get_dummies(column_choice_training['Cabin_Number_cut'], prefix = 'Cabin_num')], axis=1)
 column_choice_test = pd.concat([column_choice_test.drop(['Cabin_Number', 'Cabin_Number_cut'], axis=1),
pd.get_dummies(column_choice_test['Cabin_Number_cut'], prefix = 'Cabin_num')], axis=1)
 column_choice_training.head()
pred_glove_test_y = model.predict([test_X], batch_size=1024, verbose=1 )<set_options>
column_choice_training['Cabin_Letter'] = column_choice_training['Cabin_Letter'].replace(['T'], 'N')
 column_choice_training = pd.concat([column_choice_training.drop(['Cabin', 'Cabin_Letter'], axis=1),
pd.get_dummies(column_choice_training['Cabin_Letter'], prefix = 'Cabin_Letter')], axis=1)
 column_choice_test = pd.concat([column_choice_test.drop(['Cabin', 'Cabin_Letter'], axis=1),
pd.get_dummies(column_choice_test['Cabin_Letter'], prefix = 'Cabin_Letter')], axis=1)
 column_choice_training.head()
del word_index, embeddings_index, all_embs, embedding_matrix, model, inp, x
time.sleep(10 )<statistical_test>
column_choice_training['Family'] = column_choice_training['SibSp'] + column_choice_training['Parch']
column_choice_test['Family'] = column_choice_test['SibSp'] + column_choice_test['Parch']
column_choice_training['Family'] = column_choice_training['Family'].replace([0], 'Alone')
 column_choice_training['Family'] = column_choice_training['Family'].replace([1, 2, 3], 'Little_Family')
 column_choice_training['Family'] = column_choice_training['Family'].replace([4, 5, 6, 7, 10], 'Big_Family')
 column_choice_test['Family'] = column_choice_test['Family'].replace([0], 'Alone')
 column_choice_test['Family'] = column_choice_test['Family'].replace([1, 2, 3], 'Little_Family')
 column_choice_test['Family'] = column_choice_test['Family'].replace([4, 5, 6, 7, 10], 'Big_Family')
 print(column_choice_training['Family'].value_counts())
 print(column_choice_test['Family'].value_counts())
 column_choice_training[['Family', 'Survived']].groupby(['Family'], as_index=False ).mean().sort_values(by='Survived', ascending=False)
 column_choice_training = pd.concat([column_choice_training.drop(['SibSp', 'Parch', 'Family'], axis=1),
pd.get_dummies(column_choice_training['Family'], prefix = 'Family')], axis=1)
 column_choice_test = pd.concat([column_choice_test.drop(['SibSp', 'Parch', 'Family'], axis=1),
pd.get_dummies(column_choice_test['Family'], prefix = 'Family')], axis=1 )
EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)
 x = Attention(maxlen )(x)
 x = Dense(64, activation="relu" )(x)
 x = Dropout(0.1 )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer=Adam(lr=1e-3), metrics=['accuracy'])
 print(model.summary() )<train_model>
X_train = np.asarray(column_choice_training.drop(['Survived'], axis=1))
X_train = preprocessing.StandardScaler().fit(X_train ).transform(X_train)
 X_train[0:5]
model.fit(train_X, train_y, batch_size=512, epochs=2, validation_data=(val_X, val_y))<predict_on_test>
X_test = np.asarray(column_choice_test)
 X_test = preprocessing.StandardScaler().fit(X_test ).transform(X_test)
 X_test[0:5]
pred_fasttext_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_fasttext_val_y>thresh ).astype(int))))<predict_on_test>
y_train = np.asarray(column_choice_training['Survived'])
 y_train[0:5]
pred_fasttext_test_y = model.predict([test_X], batch_size=1024, verbose=1 )<set_options>
print('Train set:', X_train.shape,y_train.shape)
 print('Test set:', X_test.shape )
del word_index, embeddings_index, all_embs, embedding_matrix, model, inp, x
time.sleep(10 )<statistical_test>
RFC = RandomForestClassifier(oob_score = True, random_state = 1)
 param_grid = {'min_samples_leaf' : [1, 2, 4, 6, 8, 10],
'min_samples_split' : [2, 4, 6, 8, 10, 12, 14, 16],
'n_estimators': [200, 500, 800, 1100]}
param_distributions = param_grid,
n_iter = 100,
cv = cross_validation_split, verbose = 2, random_state = 0, n_jobs = -1)
 print(RS.best_score_)
 print(RS.best_params_ )
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)
 x = Attention(maxlen )(x)
 x = Dense(64, activation="relu" )(x)
 x = Dropout(0.1 )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer=Adam(lr=1e-3), metrics=['accuracy'])
 print(model.summary() )<train_model>
param_grid = { 'criterion' : ['gini', 'entropy'],
'min_samples_leaf' : [1, 2, 3],
'min_samples_split' : [3, 4, 5],
'n_estimators': [1000, 1100, 1200]}
print(GS.best_score_)
 print(GS.best_params_ )
model.fit(train_X, train_y, batch_size=512, epochs=2, validation_data=(val_X, val_y))<predict_on_test>
best_model = RandomForestClassifier(n_estimators=1000,
oob_score = True,
criterion = 'gini',
min_samples_leaf = 3,
min_samples_split = 8,
max_depth = None,
random_state = 1 ).fit(X_train,y_train)
 yhat = best_model.predict(X_train)
 print("%.4f" % best_model.oob_score_)
 importance_df = pd.concat(( pd.DataFrame(column_choice_training.drop(['Survived'], axis=1 ).columns, columns = ['variable']),
pd.DataFrame(best_model.feature_importances_, columns = ['importance'])) ,
axis = 1 ).sort_values(by='importance', ascending = False)
 importance_df
pred_paragram_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_paragram_val_y>thresh ).astype(int))))<predict_on_test>
yhat_test = best_model.predict(X_test)
 submission = testing_df.copy()
submission['Survived'] = yhat_test
submission.to_csv('submission.csv', columns=['PassengerId', 'Survived'], index=False)
 submission[['PassengerId', 'Survived']].head(15 )