Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<statistical_test>
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split, GridSearchCV, RandomizedSearchCV
from sklearn.metrics import accuracy_score, confusion_matrix, roc_auc_score, roc_curve
from sklearn.preprocessing import StandardScaler, OneHotEncoder, KBinsDiscretizer
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier, VotingClassifier
from sklearn.neural_network import MLPClassifier
import xgboost as xgb
def getGloVeEmbeddings() :
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def getFastTextEmbeddings() :
EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def getParagramEmbeddings() :
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix 
<categorify>
warnings.filterwarnings("ignore" )
glove_embedding = getGloVeEmbeddings()
fasttext_embedding = getFastTextEmbeddings()
paragram_embedding = getParagramEmbeddings()<set_options>
%matplotlib inline
class KMaxPooling(Layer):

 def __init__(self, k=1, axis=1, **kwargs):
super(KMaxPooling, self ).__init__(**kwargs)
 self.input_spec = InputSpec(ndim=3)
 self.k = k
assert axis in [1,2], 'expected dimensions(samples, filters, convolved_values),\
cannot fold along samples dimension or axis not in list [1,2]'
self.axis = axis
self.transpose_perm = [0,1,2]
self.transpose_perm[self.axis] = 2
self.transpose_perm[2] = self.axis
def compute_output_shape(self, input_shape):
input_shape_list = list(input_shape)
 input_shape_list[self.axis] = self.k
return tuple(input_shape_list)
 def call(self, x):
transposed_for_topk = tf.transpose(x, perm=self.transpose_perm)
 top_k = tf.nn.top_k(transposed_for_topk, k=self.k, sorted=True, name=None)[0]
transposed_back = tf.transpose(top_k, perm=self.transpose_perm)
 return transposed_back
class Folding(Layer):
def __init__(self, **kwargs):
super(Folding, self ).__init__(**kwargs)
 self.input_spec = InputSpec(ndim=3)
 def compute_output_shape(self, input_shape):
return(input_shape[0], input_shape[1], int(input_shape[2]/2))
def call(self, x):
input_shape = x.get_shape().as_list()
splits = tf.split(x, num_or_size_splits=int(input_shape[2]/2), axis=2)
 reduce_sums = [tf.reduce_sum(split, axis=2)for split in splits]
row_reduced = tf.stack(reduce_sums, axis=2)
 return row_reduced
def BiRNN() :
inp = Input(shape=(maxlen,))
glove = Embedding(max_features, embed_size, weights=[glove_embedding] )(inp)
 fast = Embedding(max_features, embed_size, weights=[fasttext_embedding] )(inp)
 paragram = Embedding(max_features, embed_size, weights=[paragram_embedding] )(inp)
 x = Add()([glove, fast, paragram])
 x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)
 x = GlobalMaxPool1D()(x)
 x = Dense(16, activation="relu" )(x)
 x = Dropout(0.1 )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 print(model.summary())
 return model
def MGNC_CNN() :
inp = Input(shape=(maxlen,))
glove = Embedding(max_features, embed_size, weights=[glove_embedding] )(inp)
 fast = Embedding(max_features, embed_size, weights=[fasttext_embedding] )(inp)
 paragram = Embedding(max_features, embed_size, weights=[paragram_embedding] )(inp)
 filter_sizes = [3,5]
conv_pools = []
for text_embedding in [glove,fast,paragram]:
for filter_size in filter_sizes:
l_zero = ZeroPadding1D(( filter_size-1,filter_size-1))(text_embedding)
 l_conv = Conv1D(filters=16, kernel_size=filter_size, padding='same', activation='tanh' )(l_zero)
 l_pool = GlobalMaxPool1D()(l_conv)
 conv_pools.append(l_pool)
 l_merge = Concatenate(axis=1 )(conv_pools)
 l_dense = Dense(128, activation='relu', kernel_regularizer=regularizers.l2(0.01))(l_merge)
 l_out = Dense(1, activation='sigmoid' )(l_dense)
 model = Model(inputs=inp, outputs=l_out)
 model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['acc'])
 print(model.summary())
 return model
def MV_CNN() :
inp = Input(shape=(maxlen,))
glove = Embedding(max_features, embed_size, weights=[glove_embedding] )(inp)
 fast = Embedding(max_features, embed_size, weights=[fasttext_embedding] )(inp)
 paragram = Embedding(max_features, embed_size, weights=[paragram_embedding] )(inp)
 k_top = 4
filter_sizes = [3,5]
layer_1 = []
for text_embedding in [glove, fast, paragram]:
conv_pools = []
for filter_size in filter_sizes:
l_zero = ZeroPadding1D(( filter_size-1,filter_size-1))(text_embedding)
 l_conv = Conv1D(filters=128, kernel_size=filter_size, padding='same', activation='tanh' )(l_zero)
 l_pool = KMaxPooling(k=30, axis=1 )(l_conv)
 conv_pools.append(( filter_size,l_pool))
layer_1.append(conv_pools)
 last_layer = []
for layer in layer_1:
for(filter_size, input_feature_maps)in layer:
l_zero = ZeroPadding1D(( filter_size-1,filter_size-1))(input_feature_maps)
 l_conv = Conv1D(filters=128, kernel_size=filter_size, padding='same', activation='tanh' )(l_zero)
 l_pool = KMaxPooling(k=k_top, axis=1 )(l_conv)
 last_layer.append(l_pool)
 l_merge = Concatenate(axis=1 )(last_layer)
 l_flat = Flatten()(l_merge)
 l_dense = Dense(128, activation='relu' )(l_flat)
 l_out = Dense(1, activation='sigmoid' )(l_dense)
 model = Model(inputs=inp, outputs=l_out)
 model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['acc'])
 print(model.summary())
 return model<train_on_grid>
data = pd.read_csv('/kaggle/input/titanic/train.csv')
mvcnn = MV_CNN()
mvcnn.fit(train_X, train_y, batch_size=1024, epochs=2, validation_data=(val_X, val_y))<find_best_params>
def test_thresh(lower = 0.1, upper = 0.95, jump = 0.01):
accs = {}
for i in np.arange(lower, upper, jump):
accs[i] = accuracy_score(test[1], predict(test[0], model, transformers, thresh = i, prep = False)['Survived'])
 best_thresh = np.round(sorted(accs.items() , key = lambda x: x[1], reverse = True)[0][0], 2)
 train_acc = accuracy_score(train[1], predict(train[0], model, transformers, thresh = best_thresh, prep = False)['Survived'])
 print(f'Train score: {train_acc}')
 print('Top 5 thresholds(test score):')
 print(sorted(accs.items() , key = lambda x:(1-abs(0.5-x[0])) *x[1], reverse = True)[:5])
 return best_thresh
pred_y = mvcnn.predict([val_X], batch_size=1024, verbose=1)
 max_f1 = 0
max_thresh = 0
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 score = metrics.f1_score(val_y,(pred_y > thresh ).astype(int))
print("F1 score at threshold {0} is {1}".format(thresh, score))
if score > max_f1:
max_f1 = score
max_thresh = thresh
print("** Best F1 score at threshold {0} is {1}".format(max_thresh, max_f1))
pred_mgnccnn = pred_y<save_to_csv>
def process_data(data, transformers = None):
data = data.copy()
data.set_index('PassengerId', inplace = True)
 data['title'] = data.Name.str.split(',' ).str[1].str.strip().str.split().str[0]
low_freq_titles = data.title.value_counts() [lambda x: x < 10].index
data['title'] = data['title'].apply(lambda x: 'Misc' if x in low_freq_titles else x)
 age_by_title = data.groupby('title' ).Age.median()
data['Age'] = data.apply(lambda x: age_by_title[x.title] if pd.isna(x.Age)else x.Age, axis = 1)
 data.loc[pd.isna(data.Embarked), 'Embarked'] = 'C'
data['fam_size'] = data.Parch + data.SibSp
data = data.fillna(method = 'ffill')
 vars_for_bins = ['Age', 'Fare']
vars_for_dummies = ['Sex', 'Embarked']
if transformers is not None:
bins = transformers[0]
dummy = transformers[1]
binned_age = bins.transform(np.array(data[vars_for_bins]))
dummies = dummy.transform(data[vars_for_dummies])
 else:
bins = KBinsDiscretizer()
dummy = OneHotEncoder(sparse = False, drop = 'first')
 binned_age = bins.fit_transform(np.array(data[vars_for_bins]))
dummies = dummy.fit_transform(data[vars_for_dummies])
 dummies = pd.DataFrame(dummies, columns = dummy.get_feature_names())
 binned_age = pd.DataFrame(binned_age.toarray() , columns = ['Age_'+str(i)for i in bins.bin_edges_[0]][:-1] + ['Fare_'+str(i)for i in bins.bin_edges_[1]][:-1])
 data.drop(['Cabin', 'Ticket', 'Name', 'Fare', 'Parch', 'SibSp', 'Age', 'Pclass', 'Sex', 'title', 'Embarked'], inplace = True, axis = 1, errors = 'ignore')
 data = pd.concat([data.reset_index() , dummies, binned_age], axis = 1 ).set_index('PassengerId')
 if transformers is None:
return data, bins, dummy
else:
return data
pred_y = mvcnn.predict([test_X], batch_size=1024, verbose=1)
 pred_test_y =(pred_y > max_thresh ).astype(int)
 out_df = pd.DataFrame({"qid": test_df["qid"].values})
 out_df['prediction'] = pred_test_y
out_df.to_csv("submission.csv", index=False )<load_from_csv>
def fit_model(train_data, scale = True):
preped, bins, dummy = process_data(train_data)
 scaler = StandardScaler()
X = preped.drop('Survived', axis = 1)
 if scale:
X = pd.DataFrame(scaler.fit_transform(X), columns = X.columns, index = X.index)
 else:
scaler = None
y = preped['Survived']
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3)
 print('Training Neural Net')
 best_nn = RandomizedSearchCV(MLPClassifier() ,
verbose = 0,
n_iter = 30,
param_distributions = {'hidden_layer_sizes':[(50,50,),(100,100,100,),(300,300,),(50,50,50,50,),(50,50,50,50,50,),(200,200,200,)],
'activation':['relu','tanh'],
'solver':['adam','lbfgs'],
'learning_rate':['constant','adaptive'],
'learning_rate_init': np.logspace(np.log10(0.00001), np.log(1), 20),
'max_iter':[50,100,200,500],
'random_state':[0]})
 best_nn.fit(X_train, y_train)
 best_nn = best_nn.best_estimator_
print('Training XGB')
 best_xgb = RandomizedSearchCV(xgb.XGBClassifier() , n_iter = 30,
param_distributions = {'n_estimators':np.arange(50,550,50),
'booster':['gbtree', 'gblinear','dart'],
'max_depth':np.arange(1,50),
'learning_rate': np.logspace(np.log10(0.00001), np.log(1), 20),
'reg_alpha':np.logspace(np.log10(0.00001), np.log(1), 20)})
 best_xgb.fit(X_train, y_train)
 best_xgb = best_xgb.best_estimator_
print('Training Logistic Regression')
 best_log = RandomizedSearchCV(LogisticRegression() ,
error_score=0.0,
n_iter = 30,
param_distributions = {'penalty':['l1', 'l2', 'elasticnet','none'],
'dual':[True,False],
'C': np.logspace(np.log10(0.00001), np.log(1), 20),
'fit_intercept':[True, False],
'random_state':[0],
'max_iter':[30,50,100,200,500,1000],
'solver':['newton-cg', 'lbfgs', 'liblinear', 'sag', 'saga']})
 best_log.fit(X_train, y_train)
 best_log = best_log.best_estimator_
print('Training Random Forest')
 best_forest = RandomizedSearchCV(RandomForestClassifier() , n_iter = 30, param_distributions = {'bootstrap': [True, False],
'max_depth': [10, 20, 30, 40, 50, 60, 70, 80, 90, 100, None],
'max_features': ['auto', 'sqrt'],
'min_samples_leaf': [1, 2, 4],
'min_samples_split': [2, 5, 10],
'n_estimators': [200, 400, 600, 800, 1000, 1200, 1400, 1600, 1800, 2000]})
 best_forest.fit(X_train, y_train)
 best_forest = best_forest.best_estimator_
best_estimators = [('Neural Net', best_nn),('XGB', best_xgb),('Logistic Reg', best_log),('Random Forest', best_forest)]
ensemble_model = VotingClassifier(best_estimators, voting = 'hard')
 ensemble_model.fit(X_train, y_train)
 print(f'Train Acc Score: {accuracy_score(y_train, ensemble_model.predict(X_train)) }')
 print(f'Test Acc Score: {accuracy_score(y_test, ensemble_model.predict(X_test)) }')
 return ensemble_model,(bins, dummy, scaler), preped,(X_train, y_train),(X_test, y_test )
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 print("Train shape : ",train_df.shape)
 print("Test shape : ",test_df.shape )<split>
def predict(test_df, model, transformers, thresh = 0.5, prep = True):
if prep:
temp_df = process_data(test_df, transformers)
 idx = temp_df.index
if transformers[2] is not None:
temp_df = pd.DataFrame(transformers[2].transform(temp_df), columns = temp_df.columns)
 else:
temp_df = pd.DataFrame(test_df, columns = preped.drop('Survived',axis = 1 ).columns ).copy()
idx = temp_df.index
if thresh != 0.5:
predictions =(model.predict_proba(temp_df)[:,1] > thresh)* 1
return pd.DataFrame(columns = ['PassengerId','Survived'], data = [*zip(idx, predictions)])
 else:
return pd.DataFrame(columns = ['PassengerId','Survived'], data = [*zip(idx, model.predict(temp_df)) ] )
train_df, val_df = train_test_split(train_df, test_size=0.1, random_state=2018)
 embed_size = 300
max_features = 50000
maxlen = 100<prepare_x_and_y>
model, transformers, preped, train, test = fit_model(data, scale = True )
train_X = train_df["question_text"].fillna("_na_" ).values
val_X = val_df["question_text"].fillna("_na_" ).values
test_X = test_df["question_text"].fillna("_na_" ).values<string_transform>
test_set = pd.read_csv('/kaggle/input/titanic/test.csv')
 predictions = predict(test_set, model, transformers)
 predictions.head()
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 val_X = tokenizer.texts_to_sequences(val_X)
 test_X = tokenizer.texts_to_sequences(test_X )<string_transform>
model.estimators_
train_X = pad_sequences(train_X, maxlen=maxlen)
 val_X = pad_sequences(val_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen )<prepare_x_and_y>
predictions.to_csv('submission.csv', index = False )
train_y = train_df['target'].values
val_y = val_df['target'].values<feature_engineering>
test_labels = pd.read_csv('/kaggle/input/titanic-solutions-for-selfscoring/pub')
 accuracy_score(test_labels['survived'], predictions['Survived'] )