Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<predict_on_test>
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
predictions = model.predict(test_features )<groupby>
train_data = pd.read_csv('/kaggle/input/titanic/train.csv')
 test_data = pd.read_csv('/kaggle/input/titanic/test.csv')
 train_data.head(10 )
features['winPlacePercPred'] = predictions
features['matchId'] = matches
features['groupId'] = groups
group_preds = features.groupby(['matchId', 'groupId'])['winPlacePercPred'].agg('mean' ).groupby('matchId' ).rank(pct=True ).reset_index()
group_preds = group_preds['winPlacePercPred']<sort_values>
obj_imputer = SimpleImputer(missing_values = np.nan,strategy = 'most_frequent')
 train_data['Embarked'] = obj_imputer.fit_transform(train_data[['Embarked']])
 train_data
test_data_df = test_data_df.sort_values(['matchId', 'groupId'] )<define_variables>
split_one = train_data['Name'].str.split('.', n=1, expand = True)
 train_data['Name'] = split_one[0]
split_two = train_data['Name'].str.split(',', n=1, expand = True)
 train_data['Name'] = split_two[1]
train_data['Title'] = train_data['Name'].apply(set_title)
 train_data
dictionary = dict(zip(features['groupId'].values, group_preds))<prepare_output>
col_names = ['Female','Male','St_C','St_Q','St_S']
col_names.extend(( list(train_data.columns)))
 col_names.remove('Embarked')
 col_names.remove('Sex')
 ct = ColumnTransformer(transformers=[('encoder', OneHotEncoder() , [2,7])],
remainder='passthrough')
 train_data = pd.DataFrame(data = np.array(ct.fit_transform(train_data.values)) ,
columns = col_names)
 train_data
new_ranking_preds = []
for i in test_data_df['groupId'].values:
new_ranking_preds.append(dictionary[i])
 test_data_df['winPlacePercPred'] = new_ranking_preds<prepare_output>
missing_val_data = train_data[train_data['Age'].isnull() ]
nonmissing_val_data = train_data[train_data['Age'].notnull() ]
missing_val_data.info()
predictions = pd.DataFrame(np.transpose(np.array([test_data_df.loc[:, 'Id'], test_data_df['winPlacePercPred']])))
 predictions.columns = ['Id', 'winPlacePerc']
predictions['Id'] = np.int32(predictions['Id'])
 predictions = predictions.sort_values(by=['Id'])
 predictions.head(10 )<save_to_csv>
X_MD = missing_val_data.drop(['Survived','Age'], axis = 1 ).values
y_MD = missing_val_data['Survived'].values.astype('int')
 r_state = 3
md_colnames = missing_val_data.drop(['Survived','Age'], axis = 1 ).columns
X_train, X_test, y_train, y_test = train_test_split(X_MD,
y_MD,
test_size = 0.3,
random_state = r_state)
 X_train
predictions.to_csv('PUBG_preds.csv', index=False )<import_modules>
pd.DataFrame(X_train )
import pandas as pd
import seaborn as sns
import numpy as np
import matplotlib.pyplot as plt
import statsmodels.formula.api as sm
from sklearn.model_selection import train_test_split
from sklearn.metrics import r2_score
from sklearn.ensemble import RandomForestRegressor
from sklearn import preprocessing
from scipy import stats
import lightgbm as lgb
import gc<load_from_csv>
sc_MD = StandardScaler()
X_train[:, 5:] = sc_MD.fit_transform(X_train[:, 5:])
 X_test[:, 5:] = sc_MD.transform(X_test[:, 5:])
 X_train = pd.DataFrame(X_train)
 X_test = pd.DataFrame(X_test)
 print(X_train.head(3))
print(X_test.head(3))
train_init= pd.read_csv('.. /input/train.csv' )<drop_column>
classifier_LR = LogisticRegression(random_state = r_state)
 classifier_LR.fit(X_train, y_train)
 y_pred_LR = classifier_LR.predict(X_test)
 accuracy_score(y_test, y_pred_LR )
list_largest_corr = train_init.drop(['matchId','groupId','teamKills',u'winPlacePerc'],axis=1 ).columns<merge>
classifier_KNN = KNeighborsClassifier(n_neighbors = 5,
metric = 'minkowski',
p = 2)
 classifier_KNN.fit(X_train, y_train)
 y_pred_KNN = classifier_KNN.predict(X_test)
 accuracy_score(y_test, y_pred_KNN )
train_without_out = train_init[(np.abs(stats.zscore(train_init)) < 6 ).all(axis=1)]
del train_init
gc.collect()
def add_means(train_without_out,list_largest_corr,isSampleTest=False):
agg = train_without_out.groupby(['matchId','groupId'])[list_largest_corr].agg('mean')
 agg_rank = agg.groupby('matchId')[list_largest_corr].rank(pct=True ).reset_index()
train_without_out = train_without_out.merge(agg_rank, suffixes=["", "_mean_rank"], how='left', on=['matchId', 'groupId'])
 del agg
del agg_rank
gc.collect()
agg = train_without_out.groupby(['matchId','groupId'] ).size().reset_index(name='group_size')
 train_without_out = train_without_out.merge(agg, how='left', on=['matchId', 'groupId'])
 del agg
gc.collect()
agg = train_without_out.groupby(['matchId'] ).size().reset_index(name='match_size')
 train_without_out = train_without_out.merge(agg, how='left', on=['matchId'])
 del agg
gc.collect()
agg = train_without_out.groupby(['matchId'])[list_largest_corr].agg('mean' ).reset_index()
train_without_out = train_without_out.merge(agg,suffixes=["", "_mean_match"], how='left', on=['matchId'])
 del agg
gc.collect()
if isSampleTest == False:
x = train_without_out.drop('winPlacePerc',axis=1)
 else:
x = train_without_out
scaler = preprocessing.MinMaxScaler(feature_range=(0, 1), copy=False ).fit(x)
 scaler.transform(x)
 return x
x = add_means(train_without_out,list_largest_corr)
 y = train_without_out['winPlacePerc']
del train_without_out
gc.collect()<train_model>
knn_prams = [{'n_neighbors': [1 , 2, 3, 4, 5, 6, 7],
'metric': ['minkowski'],
'p': [2]}]
grid_search_KNN = GridSearchCV(estimator = classifier_KNN,
param_grid = knn_prams,
scoring = 'accuracy',
cv = 10,
n_jobs = -1)
 grid_search_KNN.fit(X_train, y_train)
 best_accuracy_KNN = grid_search_KNN.best_score_
best_parameters_KNN = grid_search_KNN.best_params_
print("Best Accuracy: {:.2f} %".format(best_accuracy_KNN*100))
print("Best Parameters:", best_parameters_KNN )
d_train = lgb.Dataset(x, y)
 iterations = 2000
watchlist = [d_train]
params = {
'learning_rate': 0.1,
'max_depth': -1,
'num_leaves': 30,
'feature_fraction': 0.9,
'min_data_in_leaf': 100,
'lambda_l2': 4,
'objective': 'regression_l2',
'metric': 'mae',
'seed': 123}
model = lgb.train(params, train_set=d_train, num_boost_round=iterations,
valid_sets=watchlist, early_stopping_rounds=20, verbose_eval=200 )<load_from_csv>
classifier_KNN = KNeighborsClassifier(n_neighbors = 2,
metric = 'minkowski',
p = 2)
 classifier_KNN.fit(X_train, y_train)
 y_pred_KNN = classifier_KNN.predict(X_test)
 accuracy_score(y_test, y_pred_KNN )
test = pd.read_csv('.. /input/test.csv')
 x_test = add_means(test,list_largest_corr,True)
 predict = model.predict(x_test )<save_to_csv>
classifier_SVM = SVC(kernel = 'linear',
random_state = r_state)
 classifier_SVM.fit(X_train, y_train)
 y_pred_SVM = classifier_SVM.predict(X_test)
 accuracy_score(y_test, y_pred_SVM )
test['winPlacePerc'] = predict
test[['Id','winPlacePerc']].to_csv("submission.csv", index = False )<set_options>
svm_params = [{'C': [0.25, 0.5, 0.75, 0.85, 1.0],
'kernel': ['rbf'],
'gamma': [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]}]
grid_search_SVM = GridSearchCV(estimator = classifier_SVM,
param_grid = svm_params,
scoring = 'accuracy',
cv = 10,
n_jobs = -1)
 grid_search_SVM.fit(X_train, y_train)
 best_accuracy_SVM = grid_search_SVM.best_score_
best_parameters_SVM = grid_search_SVM.best_params_
print("Best Accuracy: {:.2f} %".format(best_accuracy_SVM*100))
print("Best Parameters:", best_parameters_SVM)
 classifier_KSVM = SVC(C = 0.75,
kernel = 'rbf',
random_state = r_state,
gamma = 0.3)
 classifier_KSVM.fit(X_train, y_train)
 y_pred_KSVM = classifier_KSVM.predict(X_test)
 accuracy_score(y_test, y_pred_KSVM )
%pylab inline 
<load_from_csv>
classifier_NB = GaussianNB()
classifier_NB.fit(X_train, y_train)
 y_pred_NB = classifier_NB.predict(X_test)
 accuracy_score(y_test, y_pred_NB )
data = pd.read_csv('.. /input/train.csv')
 parent_data = data.copy()
ID = data.pop('id' )<categorify>
classifier_DT = DecisionTreeClassifier(criterion = 'gini',
random_state = r_state)
 classifier_DT.fit(X_train, y_train)
 y_pred_DT = classifier_DT.predict(X_test)
 accuracy_score(y_test, y_pred_DT )
y = data.pop('species')
 y = LabelEncoder().fit(y ).transform(y)
 y_cat = to_categorical(y)
 X = StandardScaler().fit(data ).transform(data)
<train_model>
classifier_RF = RandomForestClassifier(n_estimators = 10,
criterion = 'gini',
random_state = r_state)
 classifier_RF.fit(X_train, y_train)
 y_pred_RF = classifier_RF.predict(X_test)
 accuracy_score(y_test, y_pred_RF )
model = Sequential()
model.add(Dense(250, input_dim=192, init='uniform', activation='relu'))
model.add(Dropout(0.2))
model.add(Dense(150, activation='relu'))
model.add(Dropout(0.4))
model.add(Dense(99, activation='softmax'))
model.compile(loss='categorical_crossentropy',optimizer='rmsprop', metrics = ["accuracy"])
 history = model.fit(X, y_cat, batch_size=64, nb_epoch=250, verbose=0, validation_split=0.2 )<save_to_csv>
classifier_XGB = XGBClassifier()
classifier_XGB.fit(X_train.to_numpy() , y_train)
 y_pred_XGB = classifier_XGB.predict(X_test.to_numpy())
 accuracy_score(y_test, y_pred_XGB)
test = pd.read_csv('.. /input/test.csv')
 index = test.pop('id')
 test = StandardScaler().fit(test ).transform(test)
 yPred = model.predict_proba(test)
 yPred = pd.DataFrame(yPred,index=index,columns=sort(parent_data.species.unique()))
fp = open('submission_nn_kernel.csv','w')
 fp.write(yPred.to_csv() )<import_modules>
classifier_GB = GradientBoostingClassifier()
classifier_GB.fit(X_train, y_train)
 y_pred_GB = classifier_GB.predict(X_test)
 accuracy_score(y_test, y_pred_GB )
import numpy as np
import pandas as pd
import tensorflow as tf
import math<init_hyperparams>
classifier_NN = MLPClassifier(random_state = r_state)
 classifiers = [classifier_LR,
classifier_KNN,
classifier_SVM,
classifier_KSVM,
classifier_NB,
classifier_DT,
classifier_RF,
classifier_XGB,
classifier_GB,
classifier_NN]
classifiers_names = ['Linear Regression',
'KNN',
'SVM',
'Kernel SVM',
'Naive Bayes',
'Decision Tree',
'Random Forest',
'XG Boost',
'Gradient Boost',
'NN']
accuracy_mean = []
for cl in classifiers :
accuracies = cross_val_score(estimator = cl,
X = X_train.to_numpy() ,
y = y_train,
cv = 10)
 accuracy_mean.append(accuracies.mean() *100)
 accuracy_df = pd.DataFrame({'Classifier': classifiers_names,
'Accuracy Mean': accuracy_mean})
 accuracy_df
flags = tf.app.flags
FLAGS = flags.FLAGS
flags.DEFINE_integer('num_classes', 99, 'Number of classes.')
 flags.DEFINE_integer('num_variables', 192, 'Number of variables.')
 flags.DEFINE_integer('hidden1', 2048, 'Number of units in hidden layer 1.')
 flags.DEFINE_integer('hidden2', 1024, 'Number of units in hidden layer 2.')
 flags.DEFINE_integer('num_epochs', 200, 'Number of learning epochs.')
 flags.DEFINE_integer('batch_size', 90, 'Batch size.')
 flags.DEFINE_float('keep_prob', 0.5, 'Keep probability for drop out.')
 flags.DEFINE_float('learning_rate', 0.001, 'Initial learning rate.' )<normalization>
age_missing_model1 = classifier_KSVM
age_missing_model1.fit(X_train, y_train)
 age_missing_model2 = classifier_SVM
age_missing_model2.fit(X_train, y_train)
 age_missing_model3 = classifier_NB
age_missing_model3.fit(X_train, y_train)
 voting_cl_MD = VotingClassifier(estimators = [('KSVM', age_missing_model1), 
('SVM',age_missing_model2), 
('NB',age_missing_model3)],
voting = 'hard')
 voting_cl_MD.fit(X_train, y_train)
def inference(data, data_size, keep_prob):
with tf.name_scope('hidden1'):
weights = tf.Variable(tf.truncated_normal([data_size, FLAGS.hidden1],
stddev=1.0 / math.sqrt(float(data_size))), name='weights1')
 biases = tf.Variable(tf.zeros([FLAGS.hidden1]), name='biases1')
 hidden1 = tf.nn.relu(tf.matmul(data, weights)+ biases)
 hidden1_drop = tf.nn.dropout(hidden1, keep_prob, name='layer1_dropout')
 with tf.name_scope('hidden2'):
weights = tf.Variable(tf.truncated_normal([FLAGS.hidden1, FLAGS.hidden2],
stddev=1.0 / math.sqrt(float(FLAGS.hidden1))), name='weights2')
 biases = tf.Variable(tf.zeros([FLAGS.hidden2]), name='biases2')
 hidden2 = tf.nn.sigmoid(tf.matmul(hidden1_drop, weights)+ biases)
 hidden2_drop = tf.nn.dropout(hidden2, keep_prob, name='layer2_dropout')
 with tf.name_scope('softmax_linear'):
weights = tf.Variable(tf.truncated_normal([FLAGS.hidden2, FLAGS.num_classes],
stddev=1.0 / math.sqrt(float(FLAGS.hidden2))), name='weights')
 biases = tf.Variable(tf.zeros([FLAGS.num_classes]), name='biases')
 logits = tf.matmul(hidden2_drop, weights)+ biases
return logits<compute_train_metric>
X_NMD = nonmissing_val_data.drop(['Survived'], axis = 1 ).values
y_NMD = nonmissing_val_data['Survived'].values.astype('int')
 r_state = 3
md_colnames = nonmissing_val_data.drop(['Survived'], axis = 1 ).columns
X_train, X_test, y_train, y_test = train_test_split(X_NMD,
y_NMD,
test_size = 0.3,
random_state = r_state)
 X_train
def loss(logits, labels):
labels = tf.to_int64(labels)
 cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits, labels, name='xentropy')
 loss = tf.reduce_mean(cross_entropy, name='xentropy_mean')
 return loss<train_model>
pd.DataFrame(X_train )
def training(loss):
tf.summary.scalar(loss.op.name, loss)
 optimizer = tf.train.AdamOptimizer(FLAGS.learning_rate)
 global_step = tf.Variable(0, name='global_step', trainable=False)
 train_op = optimizer.minimize(loss, global_step=global_step)
 return train_op<compute_test_metric>
sc_NMD = StandardScaler()
X_train[:, 5:] = sc_NMD.fit_transform(X_train[:, 5:])
 X_test[:, 5:] = sc_NMD.transform(X_test[:, 5:])
 X_train = pd.DataFrame(X_train,
columns = md_colnames)
 X_test = pd.DataFrame(X_test,
columns = md_colnames)
 print(X_train.head(3))
print(X_test.head(3))
def evaluation(logits, labels):
correct = tf.nn.in_top_k(logits, labels, 1)
 return tf.reduce_sum(tf.cast(correct, tf.int32))<data_type_conversions>
classifier_NN = MLPClassifier(random_state = r_state)
 classifiers = [classifier_LR,
classifier_KNN,
classifier_SVM,
classifier_KSVM,
classifier_NB,
classifier_DT,
classifier_RF,
classifier_XGB,
optimal_GB_classifier,
classifier_NN]
classifiers_names = ['Linear Regression',
'KNN',
'SVM',
'Kernel SVM',
'Naive Bayes',
'Decision Tree',
'Random Forest',
'XG Boost',
'Gradient Boost',
'NN']
accuracy_mean = []
for cl in classifiers :
accuracies = cross_val_score(estimator = cl,
X = X_train.to_numpy() ,
y = y_train,
cv = 10)
 accuracy_mean.append(accuracies.mean() *100)
 accuracy_df = pd.DataFrame({'Classifier': classifiers_names,
'Accuracy Mean': accuracy_mean})
 accuracy_df
def preprocess_data(data):
_data = data.copy()
del _data['id']
for column in _data:
if _data[column].dtypes == float:
_data[column] = z_score_normalization(_data[column])
 if 'species' in _data.columns:
_data.insert(2, 'species_cat', _data['species'].astype('category' ).cat.codes)
 _data.drop('species', axis=1, inplace=True)
 _data = _data.astype(float)
 return _data, _data.columns.size-1<load_from_csv>
test_data = pd.read_csv('/kaggle/input/titanic/test.csv')
 test_data = test_data.drop(['Ticket','Cabin'], axis = 1)
 test_data.info()
def load_data_and_labels(file, is_labels_exist=True):
df = pd.read_csv(file)
 processed_df, variables_size = preprocess_data(df)
 print('Reading %s' % file)
 print('N=%d' % len(df))
if is_labels_exist is True:
labels = list(processed_df['species_cat'])
 else:
labels = None
if is_labels_exist is True:
del processed_df['species_cat']
new_rows = []
for index, row in processed_df.iterrows() :
_list_row = []
for col in processed_df:
_list_row.append(row[col])
 new_rows.append(_list_row)
 if is_labels_exist is True:
return new_rows, labels
else:
return new_rows<compute_test_metric>
obj_imputer_test = SimpleImputer(missing_values = np.nan,
strategy = 'most_frequent')
 fare_imputer_test = SimpleImputer(missing_values = np.nan,
strategy = 'mean')
 test_data['Embarked'] = obj_imputer.fit_transform(test_data[['Embarked']])
 test_data['Fare'] = fare_imputer_test.fit_transform(test_data[['Fare']])
 test_data
def z_score_normalization(series_of_values):
_series_of_values =(series_of_values - series_of_values.mean())/ series_of_values.std()
return _series_of_values<create_dataframe>
split_one = test_data['Name'].str.split('.', n=1, expand = True)
 test_data['Name'] = split_one[0]
split_two = test_data['Name'].str.split(',', n=1, expand = True)
 test_data['Name'] = split_two[1]
test_data['Title'] = test_data['Name'].apply(set_title)
 test_data
def shuffle_data(data, labels):
new_df = pd.DataFrame(data)
 new_df['__labels__'] = labels
new_df = new_df.reindex(np.random.permutation(new_df.index))
new_labels = list(new_df['__labels__'])
 del new_df['__labels__']
new_row = []
for index, row in new_df.iterrows() :
_list_row = []
for col in new_df:
_list_row.append(row[col])
 new_row.append(_list_row)
 return new_row, new_labels<train_model>
col_names = ['Female','Male','St_C','St_Q','St_S']
col_names.extend(( list(test_data.columns)))
 col_names.remove('Embarked')
 col_names.remove('Sex')
 ct_test = ColumnTransformer(transformers=[('encoder', OneHotEncoder() , [2,7])],
remainder='passthrough')
 test_data = pd.DataFrame(data = np.array(ct_test.fit_transform(test_data.values)) ,
columns = col_names)
 test_data
def run_training(data, labels):
with tf.Graph().as_default() :
data_size = FLAGS.num_variables
num_classes = FLAGS.num_classes
data_placeholder = tf.placeholder("float", shape=(None, data_size))
labels_placeholder = tf.placeholder("int32", shape=None)
 keep_prob = tf.placeholder("float")
 logits = inference(data_placeholder, data_size, keep_prob)
 loss_op = loss(logits, labels_placeholder)
 train_op = training(loss_op)
 eval_correct = evaluation(logits, labels_placeholder)
 summary = tf.summary.merge_all()
init_op = tf.group(tf.global_variables_initializer() , tf.local_variables_initializer())
 saver = tf.train.Saver()
sess = tf.Session()
summary_writer = tf.summary.FileWriter('.', sess.graph)
 sess.run(init_op)
 for epoch in range(FLAGS.num_epochs):
for i in range(int(len(data)/FLAGS.batch_size)) :
batch = FLAGS.batch_size*i
sess.run(train_op, feed_dict={
data_placeholder: data[batch:batch + FLAGS.batch_size],
labels_placeholder: labels[batch:batch + FLAGS.batch_size],
keep_prob: FLAGS.keep_prob})
 train_accuracy = sess.run(eval_correct, feed_dict={
data_placeholder: data,
labels_placeholder: labels,
keep_prob: FLAGS.keep_prob})
 print("epoch %d, acc %g" %(epoch, train_accuracy / len(labels)))
 summary_str = sess.run(summary, feed_dict={
data_placeholder: data,
labels_placeholder: labels,
keep_prob: 1.0})
 summary_writer.add_summary(summary_str, epoch)
 data, labels = shuffle_data(data, labels)
 path = saver.save(sess, 'models.ckpt')
 print('checkpoint is saved at ' + path)
 sess.close()<load_from_csv>
missing_val_data_test = test_data[test_data['Age'].isnull() ]
nonmissing_val_data_test = test_data[test_data['Age'].notnull() ]
nonmissing_val_data_test.info()
<train_model><EOS> 
MD_ID = missing_val_data_test['PassengerId']
NMD_ID = nonmissing_val_data_test['PassengerId']
X_MD_test = missing_val_data_test.drop(['Age','PassengerId'], axis = 1 ).values
X_NMD_test = nonmissing_val_data_test.drop(['PassengerId'], axis = 1 ).values
X_MD_test[:, 5:] = sc_MD.transform(X_MD_test[:, 5:])
 X_NMD_test[:, 5:] = sc_NMD.transform(X_NMD_test[:, 5:])
 y_pred_missing_age = []
y_pred_nonmissing_age = []
y_pred_nonmissing = voting_cl_NMD.predict(X_NMD_test)
 y_pred_missing_age = voting_cl_MD.predict(X_MD_test)
 pred_df1 = pd.DataFrame(
 {"PassengerId" : MD_ID,
"Survived" : y_pred_missing_age})
 pred_df2 = pd.DataFrame(
 {"PassengerId" : NMD_ID,
"Survived" : y_pred_nonmissing})
 frames = [pred_df1, pred_df2]
result = pd.concat(frames)
 file_name = "Submission_New.csv"
result.to_csv(file_name, index = False )