Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<data_type_conversions>
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
pred_val_y =(pred_val_y > 0.50 ).astype(int )<compute_test_metric>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 data = train.append(test)
print(classification_report(y_val,pred_val_y))<compute_test_metric>
train.select_dtypes(int ).nunique()
print(confusion_matrix(y_val,pred_val_y))<predict_on_test>
test.select_dtypes(int ).nunique()
pred_test=lstm.predict([X_test], batch_size=1024, verbose=0 )<data_type_conversions>
print('Empty values by column in Train ',(train.isnull().sum()))
pred_test =(pred_test > 0.50 ).astype(int )<create_dataframe>
print('Empty values by column in Test ',(test.isnull().sum()))
out_df = pd.DataFrame({"qid":df_test["qid"].values} )<prepare_output>
data['Title'] = data['Name']
for name_string in data['Name']:
data['Title'] = data['Name'].str.extract('([A-Za-z]+)\.', expand=True)
 mapping = {'Mlle': 'Miss', 'Major': 'Mr', 'Col': 'Mr', 'Sir': 'Mr', 'Don': 'Mr', 'Mme': 'Miss',
'Jonkheer': 'Mr', 'Lady': 'Mrs', 'Capt': 'Mr', 'Countess': 'Mrs', 'Ms': 'Miss', 'Dona': 'Mrs'}
data.replace({'Title': mapping}, inplace=True)
 titles = ['Dr', 'Master', 'Miss', 'Mr', 'Mrs', 'Rev']
for title in titles:
age_to_impute = data.groupby('Title')['Age'].median() [titles.index(title)]
data.loc[(data['Age'].isnull())&(data['Title'] == title), 'Age'] = age_to_impute
train['Age'] = data['Age'][:891]
test['Age'] = data['Age'][891:]
out_df['prediction'] = pred_test<save_to_csv>
data['Family_Size'] = data['Parch'] + data['SibSp']
train['Family_Size'] = data['Family_Size'][:891]
test['Family_Size'] = data['Family_Size'][891:]
out_df.to_csv("submission.csv", index=False )<import_modules>
drop_column = [ 'Cabin', 'Ticket', 'Parch', 'SibSp', 'Name', 'Embarked']
ps = PorterStemmer()
lc = LancasterStemmer()
sb = SnowballStemmer("english")
 def read_LSTM(reset_data):
max_length = 50
if(reset_data or not os.path.exists('lstm_data_vector.npy')or not os.path.exists("lstm_labels_vector.npy")or not os.path.exists("lstm_qid_vector.npy")) :
start_time = time.time()
train = pd.read_csv('.. /input/quora-insincere-questions-classification/train.csv' ).fillna('')
 test = pd.read_csv('.. /input/quora-insincere-questions-classification/test.csv' ).fillna('')
 qid = test.drop('question_text', axis=1)
 training_labels = train['target'].to_numpy()
train_text = train['question_text']
test_text = test['question_text']
text_list = pd.concat([train_text, test_text])
 y = train['target'].values
num_train_data = y.shape[0]
start_time = time.time()
nlp = spacy.load('en_core_web_lg', disable=['parser','ner','tagger'])
 nlp.vocab.add_flag(lambda s: s.lower() in spacy.lang.en.stop_words.STOP_WORDS, spacy.attrs.IS_STOP)
 word_dict = {}
word_index = 1
lemma_dict = {}
docs = nlp.pipe(text_list, n_threads = 2)
 word_sequences = []
for doc in tqdm(docs):
word_seq = []
for token in doc:
if(token.text not in word_dict)and(token.pos_ is not "PUNCT"):
word_dict[token.text] = word_index
word_index += 1
lemma_dict[token.text] = token.lemma_
if token.pos_ is not "PUNCT":
word_seq.append(word_dict[token.text])
 word_sequences.append(word_seq)
 del docs
train_word_sequences = word_sequences[:num_train_data]
test_word_sequences = word_sequences[num_train_data:]
train_word_sequences = pad_sequences(train_word_sequences, maxlen=max_length, padding='post')
 test_word_sequences = pad_sequences(test_word_sequences, maxlen=max_length, padding='post')
 else:
padded_train = np.load("lstm_data_vector.npy")
 training_labels = np.load("lstm_data_vector.npy")
 qid = np.load("lstm_qid_vector.npy")
 padded_test = np.load("lstm_test_vector.npy")
 x_train, x_test, y_train, y_test = train_test_split(train_word_sequences, training_labels, test_size=0.01, random_state=27)
 print('x_train.shape: ', x_train.shape)
 print('x_test.shape: ', x_test.shape)
 print('y_train.shape: ', y_train.shape)
 print('y_test.shape: ', y_test.shape)
 max_question_length = 50
vocab_size = len(word_dict)
 print(vocab_size)
 return word_dict, lemma_dict, test_word_sequences, qid, x_train, x_test, y_train, y_test, vocab_size, max_question_length
def do_LSTM(word_dict, lemma_dict, actual_test, x_train, x_test, y_train, y_test, vocab_size, max_question_length):
max_length = 50
dim=300
embeddings_index={}
with zipfile.ZipFile(".. /input/quora-insincere-questions-classification/embeddings.zip")as zf:
with io.TextIOWrapper(zf.open("glove.840B.300d/glove.840B.300d.txt"), encoding="utf-8")as f:
for line in tqdm(f):
values=line.split(' ')
 word=values[0]
vectors=np.asarray(values[1:],'float32')
 embeddings_index[word]=vectors
embeddings_index = dict(embeddings_index)
 print('embedding stuff')
 embed_size = 300
embedding_matrix = np.zeros(( len(word_dict)+1, embed_size), dtype=np.float32)
 empty = np.zeros(( embed_size,), dtype=np.float32)- 1.
 for key in tqdm(word_dict):
word = key
embedding_vector1 = embeddings_index.get(word)
 word = key.lower()
embedding_vector2 = embeddings_index.get(word)
 word = key.upper()
embedding_vector3 = embeddings_index.get(word)
 word = key.capitalize()
embedding_vector4 = embeddings_index.get(word)
 word = ps.stem(key)
 embedding_vector5 = embeddings_index.get(word)
 word = lemma_dict[key]
embedding_vector6 = embeddings_index.get(word)
 if embedding_vector1 is not None:
embedding_matrix[word_dict[key]] = embedding_vector1
continue
if embedding_vector2 is not None:
embedding_matrix[word_dict[key]] = embedding_vector2
continue
if embedding_vector3 is not None:
embedding_matrix[word_dict[key]] = embedding_vector3
continue
if embedding_vector4 is not None:
embedding_matrix[word_dict[key]] = embedding_vector4
continue
if embedding_vector5 is not None:
embedding_matrix[word_dict[key]] = embedding_vector5
continue
if embedding_vector6 is not None:
embedding_matrix[word_dict[key]] = embedding_vector6
continue
embedding_matrix[word_dict[key]] = empty
embedding_vecor_length = 50
model = Sequential()
model.add(Embedding(vocab_size+1, 300, weights=[embedding_matrix], input_length=max_length, trainable=False))
model.add(LSTM(100))
model.add(Dense(1, activation='sigmoid'))
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 print(model.summary())
 model.fit(x_train, y_train, epochs=3, batch_size=512)
 print("starting predict")
 start = time.time()
predictions = model.predict_classes(x_test)
 print('Time to predict: ', time.time() - start)
 print("starting Real predict")
 start = time.time()
actual_pred = model.predict_classes(actual_test)
 print('Time to real predict: ', time.time() - start)
 return actual_pred, predictions
def main() :
max_length = 55
lstm = True
reset_data = True
if lstm:
word_dict, lemma_dict, actual_test, qid, x_train, x_test, y_train, y_test, vocab_size, max_question_length = read_LSTM(reset_data)
 else:
return
print("starting training")
 start = time.time()
actual_pred, tes = do_LSTM(word_dict, lemma_dict, actual_test, x_train, x_test, y_train, y_test, vocab_size, max_question_length)
 print('Time to train & predict: ', time.time() - start)
 print("confusion matrix:
", confusion_matrix(y_test, tes))
print("accuracy: ", accuracy_score(y_test, tes))
print("f1: ", f1_score(y_test, tes, average='macro'))
print("classification report:
", classification_report(y_test, tes))
print("precision: ", precision_score(y_test, tes))
print("recall: ", recall_score(y_test, tes))
print(actual_pred)
 qid['prediction'] = actual_pred
qid.to_csv('submission.csv', index=False)
 if __name__ == "__main__":
main()<import_modules>
train.drop(drop_column, axis = 1, inplace = True)
 test.drop(drop_column, axis = 1, inplace = True)
import pandas as pd
import numpy as np
from sklearn.model_selection import StratifiedKFold
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.layers import *
from keras.models import *
from keras import initializers, regularizers, constraints, optimizers, layers
from keras.initializers import *
from keras.optimizers import *
import keras.backend as K
from keras.callbacks import *
import tensorflow as tf
import os
import gc
import re
from unidecode import unidecode<load_from_csv>
mapping = {'male':1, 'female':0}
train['Sex'] = train['Sex'].replace(mapping ).astype(np.float64)
 test['Sex'] = test['Sex'].replace(mapping ).astype(np.float64 )
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv")
 print("Train shape : ", train.shape)
 print("Test shape : ", test.shape )<feature_engineering>
test['Fare'].fillna(value = test['Fare'].mode() [0], inplace = True)
train["question_text"] = train["question_text"].str.lower()
test["question_text"] = test["question_text"].str.lower()
puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…',
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─',
'▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]
def clean_text(x):
x = str(x)
 for punct in puncts:
x = x.replace(punct, f' {punct} ')
 return x
train["question_text"] = train["question_text"].apply(lambda x: clean_text(x))
test["question_text"] = test["question_text"].apply(lambda x: clean_text(x))<prepare_x_and_y>
def detect_outliers(df,n,features):
outlier_indices = []
for col in features:
Q1 = np.percentile(df[col], 25)
 Q3 = np.percentile(df[col],75)
 IQR = Q3 - Q1
outlier_step = 1.5 * IQR
outlier_list_col = df[(df[col] < Q1 - outlier_step)|(df[col] > Q3 + outlier_step)].index
outlier_indices.extend(outlier_list_col)
 outlier_indices = Counter(outlier_indices)
 multiple_outliers = list(k for k, v in outlier_indices.items() if v > n)
 return multiple_outliers
Outliers_to_drop = detect_outliers(train,2,[["Pclass", "Sex", "Age", "Fare","Family_Size","Survived"]])
 train.loc[Outliers_to_drop]
train = train.drop(Outliers_to_drop, axis = 0 ).reset_index(drop=True )
embed_size = 300
max_features = None
maxlen = 72
X = train["question_text"].fillna(" " ).values
X_test = test["question_text"].fillna(" " ).values
tokenizer = Tokenizer(num_words=max_features, filters='')
 tokenizer.fit_on_texts(list(X))
X = tokenizer.texts_to_sequences(X)
 X_test = tokenizer.texts_to_sequences(X_test)
 X = pad_sequences(X, maxlen=maxlen)
 X_test = pad_sequences(X_test, maxlen=maxlen)
 Y = train['target'].values
sub = test[['qid']]<set_options>
features = ["Pclass", "Sex", "Age", "Fare", "Family_Size"]
X_train = train[features]
y_train = train["Survived"]
X_test = test[features]
del train, test
gc.collect()<statistical_test>
X_training, X_valid, y_training, y_valid = train_test_split(X_train, y_train, test_size=0.2, random_state=0)
word_index = tokenizer.word_index
max_features = len(word_index)+1
def load_glove(word_index):
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if o.split(" ")[0] in word_index)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
embedding_matrix = np.random.normal(emb_mean, emb_std,(max_features, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_para(word_index):
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100 and o.split(" ")[0] in word_index)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
embedding_matrix = np.random.normal(emb_mean, emb_std,(max_features, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix<normalization>
from sklearn.model_selection import cross_val_score
from sklearn.metrics import f1_score, make_scorer
from sklearn.ensemble import RandomForestClassifier
embedding_matrix_1 = load_glove(word_index)
 embedding_matrix_2 = load_para(word_index)
 embedding_matrix = np.mean(( embedding_matrix_1, embedding_matrix_2), axis=0)
 del embedding_matrix_1, embedding_matrix_2
gc.collect()
np.shape(embedding_matrix )<normalization>
scorer = make_scorer(f1_score, greater_is_better=True, average = 'macro')
 warnings.filterwarnings('ignore', category = ConvergenceWarning)
 warnings.filterwarnings('ignore', category = DeprecationWarning)
 warnings.filterwarnings('ignore', category = UserWarning )
def squash(x, axis=-1):
s_squared_norm = K.sum(K.square(x), axis, keepdims=True)
 scale = K.sqrt(s_squared_norm + K.epsilon())
 return x / scale
class Capsule(Layer):
def __init__(self, num_capsule, dim_capsule, routings=3, kernel_size=(9, 1), share_weights=True,
activation='default', **kwargs):
super(Capsule, self ).__init__(**kwargs)
 self.num_capsule = num_capsule
self.dim_capsule = dim_capsule
self.routings = routings
self.kernel_size = kernel_size
self.share_weights = share_weights
if activation == 'default':
self.activation = squash
else:
self.activation = Activation(activation)
 def build(self, input_shape):
super(Capsule, self ).build(input_shape)
 input_dim_capsule = input_shape[-1]
if self.share_weights:
self.W = self.add_weight(name='capsule_kernel',
shape=(1, input_dim_capsule,
self.num_capsule * self.dim_capsule),
initializer='glorot_uniform',
trainable=True)
 else:
input_num_capsule = input_shape[-2]
self.W = self.add_weight(name='capsule_kernel',
shape=(input_num_capsule,
input_dim_capsule,
self.num_capsule * self.dim_capsule),
initializer='glorot_uniform',
trainable=True)
 def call(self, u_vecs):
if self.share_weights:
u_hat_vecs = K.conv1d(u_vecs, self.W)
 else:
u_hat_vecs = K.local_conv1d(u_vecs, self.W, [1], [1])
 batch_size = K.shape(u_vecs)[0]
input_num_capsule = K.shape(u_vecs)[1]
u_hat_vecs = K.reshape(u_hat_vecs,(batch_size, input_num_capsule,
self.num_capsule, self.dim_capsule))
u_hat_vecs = K.permute_dimensions(u_hat_vecs,(0, 2, 1, 3))
b = K.zeros_like(u_hat_vecs[:, :, :, 0])
 for i in range(self.routings):
b = K.permute_dimensions(b,(0, 2, 1))
c = K.softmax(b)
 c = K.permute_dimensions(c,(0, 2, 1))
b = K.permute_dimensions(b,(0, 2, 1))
outputs = self.activation(tf.keras.backend.batch_dot(c, u_hat_vecs, [2, 2]))
if i < self.routings - 1:
b = tf.keras.backend.batch_dot(outputs, u_hat_vecs, [2, 3])
 return outputs
def compute_output_shape(self, input_shape):
return(None, self.num_capsule, self.dim_capsule )<set_options>
model_results = pd.DataFrame(columns = ['model', 'cv_mean', 'cv_std'])
 def cv_model(X_train, y_train, model, name, model_results = None, sort = True):
cv_scores = cross_val_score(model, X_train, y_train, cv = 10,
scoring = scorer, n_jobs = -1)
 print('Mean ', round(cv_scores.mean() , 5),'STd', round(cv_scores.std() , 5))
if model_results is not None:
model_results = model_results.append(pd.DataFrame({'model':name,
'cv_mean': cv_scores.mean() ,
'cv_std' : cv_scores.std() },
index = [0]), ignore_index = True)
 return model_results
model_results = cv_model(X_train, y_train, LinearSVC() , 'LSVC',
model_results)
 model_results = cv_model(X_train, y_train,
GaussianNB() , 'GNB', model_results)
 model_results = cv_model(X_train, y_train,
MLPClassifier(hidden_layer_sizes=(16, 32, 64, 64, 32)) ,
'MLP', model_results)
 model_results = cv_model(X_train, y_train,
LinearDiscriminantAnalysis() ,
'LDA', model_results)
 model_results = cv_model(X_train, y_train,
RidgeClassifierCV() , 'RIDGE', model_results)
 for n in [5, 10, 20]:
print('
KNN with {n} neighbors
', n)
 model_results = cv_model(X_train, y_train,
KNeighborsClassifier(n_neighbors = n),
n, model_results)
 model_results = cv_model(X_train, y_train,
ExtraTreesClassifier(n_estimators = 100, random_state = 10),
'EXT', model_results)
 model_results = cv_model(X_train, y_train,
RandomForestClassifier(100, random_state=10),
'RF', model_results)
 model_results = cv_model(X_train, y_train,
XGBClassifier(n_estimators= 100, random_state=10),
'XGB', model_results)
 model_results.set_index('model', inplace = True )
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim<choose_model_class>
xg_clf = XGBClassifier()
parameters_rf = {'n_estimators' : [200],'learning_rate': [0.1],
'max_depth': [4], "min_child_weight":[6], "gamma":[0], "subsample":[0.80]
}
grid_rf = GridSearchCV(xg_clf, parameters_rf, scoring=make_scorer(accuracy_score))
grid_rf.fit(X_training, y_training)
 xg_clf = grid_rf.best_estimator_
pred_xg = xg_clf.predict(X_valid)
 acc_xg = accuracy_score(y_valid, pred_xg)
 print("The Score for XGBoost is: " + str(acc_xg))
print("The best Score for XGBoost is: " + str(grid_rf.best_score_))
def capsule() :
K.clear_session()
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = SpatialDropout1D(rate=0.22 )(x)
 x = Bidirectional(CuDNNGRU(100, return_sequences=True,))(x)
 x = Capsule(num_capsule=10, dim_capsule=10, routings=4, share_weights=True )(x)
 x = Attention(10 )(x)
 x = Dense(100, activation="relu" )(x)
 x = Dropout(0.22 )(x)
 x = BatchNormalization()(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer=Adam())
 return model<compute_test_metric>
rf_clf = RandomForestClassifier()
parameters_rf = {"n_estimators": [4, 5, 6, 7, 8, 9, 10, 15], "criterion": ["gini", "entropy"], "max_features": ["auto", "sqrt", "log2"],
"max_depth": [2, 3, 5, 10], "min_samples_split": [2, 3, 5, 10]}
grid_rf = GridSearchCV(rf_clf, parameters_rf, scoring=make_scorer(accuracy_score))
grid_rf.fit(X_training, y_training)
 rf_clf = grid_rf.best_estimator_
rf_clf.fit(X_training, y_training)
 pred_rf = rf_clf.predict(X_valid)
 acc_rf = accuracy_score(y_valid, pred_rf)
 print("The Score for Random Forest is: " + str(acc_rf))
print("The best Score for Random Forest is: " + str(grid_rf.best_score_))
<split><EOS> 
submission_predictions = rf_clf.predict(X_test)
 submission = pd.DataFrame({"PassengerId": test["PassengerId"],
"Survived": submission_predictions})
 submission.to_csv("titanicprediction.csv", index=False)
 print(submission.shape )