Kaggle competition: Titanic - Machine Learning from Disaster
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub.prediction = test_preds > search_result['threshold']
sub.to_csv("submission.csv", index=False )<import_modules>
pd.set_option("display.max_rows",200)
 pd.set_option("display.max_columns",200)
 %matplotlib inline
sns.set_style('whitegrid')
tqdm.pandas() 
<load_from_csv>
df_train = pd.read_csv('.. /input/train.csv')
 df_test = pd.read_csv('.. /input/test.csv')
 print(len(df_train),len(df_test))
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv")
 print(f"Train shape: {train.shape}")
 print(f"Test shape: {test.shape}")
 train.sample()<split>
df_test.head()
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr):
return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))<feature_engineering>
len(df_train[df_train['Name'].str.contains('Dr.')] )
def check_coverage(vocab,embeddings_index):
a, oov, k, i = {}, {}, 0, 0
for word in vocab:
try:
a[word] = embeddings_index[word]
k += vocab[word]
except:
oov[word] = vocab[word]
i += vocab[word]
pass
print(f'Found embeddings for {(len(a)/ len(vocab)) :.2%} of vocab')
 print(f'Found embeddings for {(k /(k + i)) :.2%} of all text')
 sorted_x = sorted(oov.items() , key=(lambda x: x[1]), reverse=True)
 return sorted_x
def get_vocab(question_series):
sentences = question_series.str.split().values
words = [item for sublist in sentences for item in sublist]
return dict(Counter(words))<split>
len(df_train[df_train['Name'].str.contains('Sir.')] )
vocab = get_vocab(train["question_text"])
 out_of_vocab = check_coverage(vocab, embeddings_index)
 out_of_vocab[:10]<string_transform>
len(df_train[df_train['Cabin'].isnull() ])
punct = set('?!.,"
embed_punct = punct & set(embeddings_index.keys())
 def clean_punctuation(txt):
for p in "/-":
txt = txt.replace(p, ' ')
 for p in "'`‘":
txt = txt.replace(p, '')
 for p in punct:
txt = txt.replace(p, f' {p} ' if p in embed_punct else ' _punct_ ')
 return txt<feature_engineering>
df_train.drop(['Name','PassengerId','Ticket','Cabin'],axis=1,inplace=True)
 df_train.head()
train["question_text"] = train["question_text"].map(lambda x: clean_punctuation(x)).str.replace('\d+', '
test["question_text"] = test["question_text"].map(lambda x: clean_punctuation(x)).str.replace('\d+', '
vocab = get_vocab(train["question_text"])
 out_of_vocab = check_coverage(vocab, embeddings_index )<string_transform>
df_train[df_train['Embarked'].isnull() ]
train, validation = train_test_split(train, test_size=0.08, random_state=20181224)
 embed_size = 300
vocab_size = 95000
maxlen = 100
train_X = train["question_text"].fillna("_
val_X = validation["question_text"].fillna("_
test_X = test["question_text"].fillna("_
tokenizer = Tokenizer(num_words=vocab_size, filters='', lower=False)
 tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 val_X = tokenizer.texts_to_sequences(val_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 train_X = pad_sequences(train_X, maxlen=maxlen)
 val_X = pad_sequences(val_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 train_y = train['target'].values
val_y = validation['target'].values<define_variables>
df_train['Embarked'].value_counts()
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]<feature_engineering>
df_train['Embarked'].fillna('S',inplace=True)
 df_train[df_train['Embarked'].isnull() ]
word_index = tokenizer.word_index
nb_words = min(vocab_size, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
num_missed = 0
for word, i in word_index.items() :
if i >= vocab_size: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
else: num_missed += 1
print(num_missed )<prepare_x_and_y>
df_train = pd.concat([df_train,pd.get_dummies(df_train['Embarked'],prefix='Embarked')],axis=1)
 df_train.drop('Embarked',axis=1,inplace=True)
 df_train.head()
hidden_layer_size = 100
BATCH_SIZE = 64
tf.reset_default_graph()
X = tf.placeholder(tf.int32, [None, maxlen], name='X')
 Y = tf.placeholder(tf.float32, [None], name='Y')
 batch_size = tf.placeholder(tf.int64 )<define_variables>
df_train[df_train['Fare'].isnull() ]
dataset = tf.data.Dataset.from_tensor_slices(( X, Y)).shuffle(buffer_size=1000 ).batch(batch_size ).repeat()
test_dataset = tf.data.Dataset.from_tensor_slices(( X, Y)).batch(batch_size)
 iterator = tf.data.Iterator.from_structure(dataset.output_types,
dataset.output_shapes)
 train_init_op = iterator.make_initializer(dataset)
 test_init_op = iterator.make_initializer(test_dataset)
 questions, labels = iterator.get_next()<categorify>
avg_fare = pd.DataFrame([fare_notsurv.mean() ,fare_surv.mean() ])
 std_fare = pd.DataFrame([fare_notsurv.std() ,fare_surv.std() ])
 print("Mean fare for not survived is {} and survived is {}".format(fare_notsurv.mean() ,\
fare_surv.mean()))
embeddings = tf.get_variable(name="embeddings", shape=embedding_matrix.shape,
initializer=tf.constant_initializer(np.array(embedding_matrix)) ,
trainable=False)
 embed = tf.nn.embedding_lookup(embeddings, questions )<choose_model_class>
df_train[df_train['Age'].isnull() ]
lstm_cell= tf.nn.rnn_cell.LSTMCell(hidden_layer_size)
 _, final_state = tf.nn.dynamic_rnn(lstm_cell, embed, dtype=tf.float32)
 last_layer = tf.layers.dense(final_state.h, 1)
 prediction = tf.nn.sigmoid(last_layer)
 prediction = tf.squeeze(prediction, [1] )<choose_model_class>
avg_age = pd.DataFrame([age_notsurv.mean() ,age_surv.mean() ])
 std_age = pd.DataFrame([age_notsurv.std() ,age_surv.std() ])
 print("Mean age for not survived is {} and survived is {}".format(age_notsurv.mean() ,\
age_surv.mean()))
learning_rate=0.001
loss = tf.nn.sigmoid_cross_entropy_with_logits(logits=tf.squeeze(last_layer), labels=labels)
 loss = tf.reduce_mean(loss)
 optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate ).minimize(loss )<compute_train_metric>
df_train['Family'] = df_train['Parch'] + df_train['SibSp']
df_train.drop(['Parch','SibSp'],axis=1,inplace=True )
with tf.name_scope('metrics'):
F1, f1_update = tf.contrib.metrics.f1_score(labels=labels, predictions=prediction, name='my_metric')
 running_vars = tf.get_collection(tf.GraphKeys.LOCAL_VARIABLES, scope="my_metric")
 reset_op = tf.variables_initializer(var_list=running_vars )<init_hyperparams>
df_train['Family'].loc[df_train['Family']>1]=1
df_train['Family'].loc[df_train['Family']==0]=0
df_train['Family'].value_counts()
num_epochs = 10
seed = 3
sess = tf.Session()
sess.run(tf.global_variables_initializer())
 sess.run(tf.local_variables_initializer())
 costs, f1 = [], []<define_variables>
def get_person(passenger):
age,sex=passenger
return 'child' if age < 12 else sex
df_train['Person'] = df_train[['Age','Sex']].apply(get_person,axis=1)
 df_train.drop('Sex',axis=1,inplace=True )
start = time.time()
end = 0
max_time = 6600
sess.run(train_init_op, feed_dict={X:train_X, Y:train_y, batch_size:BATCH_SIZE})
 num_iter = 1000
num_batches = int(train_X.shape[0] / BATCH_SIZE)
 for epoch in range(1, num_epochs+1):
seed += seed
tf.set_random_seed(seed)
 iter_cost = 0.
 prev_iter = 0.
 for i in range(num_batches):
_ , batch_loss, _ = sess.run([optimizer, loss, f1_update])
 iter_cost += batch_loss
end = time.time()
if(end-start > max_time):
break
if(i % num_iter == 0 and i > 0):
iter_cost /=(i-prev_iter)
 prev_iter = i
cur_f1 = sess.run(F1)
 sess.run(reset_op)
 f1.append(cur_f1)
 costs.append(iter_cost)
 print(f"Epoch {epoch} Iteration {i:5} cost: {iter_cost:.10f} f1: {cur_f1:.10f} time: {end-start:4.4f}")
 batch_cost = 0 .<train_model>
df_train = pd.concat([df_train,pd.get_dummies(df_train['Person'],prefix='Person')],axis=1)
 df_train.drop(['Person'],axis=1,inplace=True )
sz = 90
tf.set_random_seed(2018)
 sess.run(test_init_op, feed_dict={X: val_X, Y: val_y, batch_size: sz})
 val_pred = np.concatenate([sess.run(prediction)for _ in range(int(val_X.shape[0]/sz)) ] )<compute_test_metric>
df_test.drop(['Name','Ticket','Cabin'],axis=1,inplace=True)
 df_test['Embarked'].fillna('S',inplace=True)
 df_test = pd.concat([df_test,pd.get_dummies(df_test['Embarked'],prefix='Embarked')],axis=1)
 df_test.drop('Embarked',axis=1,inplace=True)
 df_test['Fare'].fillna(df_test['Fare'].mean() ,inplace=True)
 df_test['Age'].fillna(df_test['Age'].dropna().median() ,inplace=True)
 df_test['Family'] = df_test['Parch'] + df_test['SibSp']
df_test.drop(['Parch','SibSp'],axis=1,inplace=True)
 df_test['Family'].loc[df_test['Family']>1]=1
df_test['Family'].loc[df_test['Family']==0]=0
def get_person(passenger):
age,sex=passenger
return 'child' if age < 12 else sex
df_test['Person'] = df_test[['Age','Sex']].apply(get_person,axis=1)
 df_test.drop('Sex',axis=1,inplace=True)
 df_test = pd.concat([df_test,pd.get_dummies(df_test['Person'],prefix='Person')],axis=1)
 df_test.drop(['Person'],axis=1,inplace=True)
 df_test.head()
thresh = thresholds[np.argmax(scores)]
print(f"Best Validation F1 Score is {max(scores):.4f} at threshold {thresh}" )<split>
features_train = df_train.drop(['Survived'],axis=1)
 target_train= df_train['Survived']
features_test = df_test.drop(['PassengerId'],axis=1 )
sz=30
temp_y = val_y[:test_X.shape[0]]
sub = test[['qid']]
sess.run(test_init_op, feed_dict={X: test_X, Y: temp_y, batch_size:sz})
 sub['prediction'] = np.concatenate([sess.run(prediction)for _ in range(int(test_X.shape[0]/sz)) ] )<save_to_csv>
train_x,test_x,train_y,test_y = train_test_split(features_train,target_train,test_size=0.2,random_state=42)
sub['prediction'] =(sub['prediction'] > thresh ).astype(np.int16)
 sub.to_csv("submission.csv", index=False)
 sub.sample()<define_variables>
clf_nb = GaussianNB()
clf_nb.fit(features_train,target_train)
 target_test_nb = clf_nb.predict(features_test )
MAX_SEQUENCE_LENGTH = 60
MAX_WORDS = 45000
EMBEDDINGS_LOADED_DIMENSIONS = 300<load_from_csv>
df_test['Survived'] = target_test_nb
df_test[['PassengerId','Survived']].to_csv('gaussnb-kaggle.csv',index=False )
df_train = pd.read_csv(".. /input/train.csv")
 df_test = pd.read_csv(".. /input/test.csv" )<define_variables>
clf_nb.fit(train_x,train_y)
 pred_gnb_y = clf_nb.predict(test_x)
 print('Accuracy score of Gaussian NB is {}'.format(metrics.accuracy_score(pred_gnb_y,test_y)) )
BATCH_SIZE = 512
Q_FRACTION = 1
questions = df_train.sample(frac=Q_FRACTION)
 question_texts = questions["question_text"].values
question_targets = questions["target"].values
test_texts = df_test["question_text"].fillna("_na_" ).values
print(f"Working on {len(questions)} questions" )<load_pretrained>
svc = SVC(kernel='rbf',class_weight='balanced')
 param_grid_svm = {'C': [1, 5, 10, 50,100],
'gamma': [0.0001, 0.0005, 0.001, 0.005,0.01]}
grid_svm = GridSearchCV(estimator=svc, param_grid=param_grid_svm)
 grid_svm.fit(train_x,train_y)
 grid_svm.best_params_
def load_embeddings(file):
embeddings = {}
with open(file)as f:
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings = dict(get_coefs(*line.split(" ")) for line in f)
 print('Found %s word vectors.' % len(embeddings))
return embeddings
%time pretrained_embeddings = load_embeddings(".. /input/embeddings/glove.840B.300d/glove.840B.300d.txt" )<string_transform>
clf_svm = grid_svm.best_estimator_
clf_svm.fit(features_train,target_train)
 target_test_svm = clf_svm.predict(features_test )
tokenizer = Tokenizer(num_words=MAX_WORDS)
 %time tokenizer.fit_on_texts(list(df_train["question_text"].values))<categorify>
df_test['Survived'] = target_test_svm
df_test[['PassengerId','Survived']].to_csv('svm-kaggle.csv',index=False )
def create_embedding_weights(tokenizer, embeddings, dimensions):
not_embedded = defaultdict(int)
 word_index = tokenizer.word_index
words_count = min(len(word_index), MAX_WORDS)
 embeddings_matrix = np.zeros(( words_count, dimensions))
for word, i in word_index.items() :
if i >= MAX_WORDS:
continue
embedding_vector = embeddings.get(word)
 if embedding_vector is not None:
embeddings_matrix[i] = embedding_vector
return embeddings_matrix
pretrained_emb_weights = create_embedding_weights(tokenizer, pretrained_embeddings, EMBEDDINGS_LOADED_DIMENSIONS )<compute_train_metric>
clf_svm.fit(train_x,train_y)
 pred_svm_y = clf_svm.predict(test_x)
 print('Accuracy score of SVM is {}'.format(metrics.accuracy_score(pred_svm_y,test_y)) )
THRESHOLD = 0.35
class EpochMetricsCallback(keras.callbacks.Callback):
def on_train_begin(self, logs={}):
self.f1s = []
self.precisions = []
self.recalls = []
def on_epoch_end(self, epoch, logs={}):
predictions = self.model.predict(self.validation_data[0])
 predictions =(predictions > THRESHOLD ).astype(int)
 predictions = np.asarray(predictions)
 targets = self.validation_data[1]
f1 = metrics.f1_score(targets, predictions)
 precision = metrics.precision_score(targets, predictions)
 recall = metrics.recall_score(targets, predictions)
 print(" - F1 score: {0:.4f}, Precision: {1:.4f}, Recall: {2:.4f}" 
.format(f1, precision, recall))
self.f1s.append(f1)
 self.precisions.append(precision)
 self.recalls.append(recall)
 return
def display_model_history(history):
data = pd.DataFrame(data={'Train': history.history['loss'], 'Test': history.history['val_loss']})
 ax = sns.lineplot(data=data, palette="pastel", linewidth=2.5, dashes=False)
 ax.set(xlabel='Epoch', ylabel='Loss', title='Loss')
 sns.despine()
plt.show()
def display_model_epoch_metrics(epoch_callback):
data = pd.DataFrame(data = {
'F1': epoch_callback.f1s,
'Precision': epoch_callback.precisions,
'Recall': epoch_callback.recalls})
 sns.lineplot(data=data, palette='muted', linewidth=2.5, dashes=False)
 sns.despine()
plt.show() 
<prepare_x_and_y>
rf = RandomForestClassifier(criterion='entropy')
 param_grid_rf = {'n_estimators':[10,100,250,500,1000],
'max_features':['sqrt','log2'],'min_samples_split':[2,5,10,50,100]}
grid_rf = GridSearchCV(estimator=rf,param_grid=param_grid_rf)
 grid_rf.fit(train_x,train_y)
 grid_rf.best_params_
%time X = pad_sequences(tokenizer.texts_to_sequences(question_texts), maxlen=MAX_SEQUENCE_LENGTH)
 %time Y = question_targets
%time test_word_tokens = pad_sequences(tokenizer.texts_to_sequences(test_texts), maxlen=MAX_SEQUENCE_LENGTH )<choose_model_class>
clf_rf = grid_rf.best_estimator_
clf_rf.fit(features_train,target_train)
 target_test_rf = clf_rf.predict(features_test )
def make_model() :
tokenized_input = Input(shape=(MAX_SEQUENCE_LENGTH,), name="tokenized_input")
 embedding = Embedding(MAX_WORDS, EMBEDDINGS_LOADED_DIMENSIONS,
weights=[pretrained_emb_weights],
trainable=False )(tokenized_input)
 d0 = SpatialDropout1D(0.1 )(embedding)
 lstm = Bidirectional(LSTM(128, return_sequences=True))(d0)
 lstm = Bidirectional(LSTM(64, return_sequences=False))(lstm)
 d1 = Dropout(0.15 )(lstm)
 d1 = Dense(64 )(d1)
 b = BatchNormalization()(d1)
 out = Dense(1, activation='sigmoid' )(b)
 model = Model(inputs=[tokenized_input], outputs=out)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 model.summary()
return model<train_model>
df_test['Survived'] = target_test_rf
df_test[['PassengerId','Survived']].to_csv('rf-kaggle.csv',index=False )
train_X, test_X, train_Y, test_Y = train_test_split(X, Y, test_size=0.01)
 epoch_callback = EpochMetricsCallback()
model = make_model()
history = model.fit(x=train_X, y=train_Y, validation_split=0.015,
batch_size=BATCH_SIZE, epochs=4, verbose=2,
callbacks=[epoch_callback] )<save_to_csv>
clf_rf.fit(train_x,train_y)
 pred_rf_y = clf_rf.predict(test_x)
 print('Accuracy score of RF is {}'.format(metrics.accuracy_score(pred_rf_y,test_y)) )
test_word_tokens = pad_sequences(tokenizer.texts_to_sequences(test_texts), maxlen=MAX_SEQUENCE_LENGTH)
 kaggle_predictions =(model.predict([test_word_tokens], batch_size=1024, verbose=2))
df_out = pd.DataFrame({"qid":df_test["qid"].values})
 df_out['prediction'] =(kaggle_predictions > THRESHOLD ).astype(int)
 df_out.to_csv("submission.csv", index=False )<categorify>
target_avg = 0.2 * target_test_nb + 0.3 * target_test_svm + 0.5 * target_test_rf
df_test['Survived'] = target_test_rf
df_test[['PassengerId','Survived']].to_csv('avg-kaggle.csv',index=False )