Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<find_best_params>
trainData = pd.read_csv('.. /input/train.csv')
 testData = pd.read_csv('.. /input/test.csv')
 trainData.head(5 )
find_c(year_list, None, np.linspace(0.5, 2, 16))<compute_test_metric>
print("Number of non-null values in each column in dataset")
 print("Survived = ", trainData['Survived'].count())
 print("Pclass = ", trainData['Pclass'].count())
 print("Sex = ", trainData['Sex'].count())
 print("Age = ", trainData['Age'].count())
 print("SibSp = ", trainData['SibSp'].count())
 print("Parch = ", trainData['Parch'].count())
 print("Ticket = ", trainData['Ticket'].count())
 print("Fare = ", trainData['Fare'].count())
 print("Cabin = ", trainData['Cabin'].count())
 print("Embarked = ", trainData['Embarked'].count() )
def find_w(year_list, w_list, c_list):
loss_array = cv_w_c(year_list, w_list, c_list)
 a = divmod(loss_array.argmin() , loss_array.shape[-1])
 print('w_list={}'.format(w_list))
print('w={} c={} loss={}'.format(w_list[a[0]], c_list[a[1]], loss_array[a]))
plt.plot(w_list, loss_array[:, a[1]])
 plt.show()<define_variables>
def logisticRegression(X, Y, Xtest, alpha, nEpoch, sh, lamb=0):
x = tf.placeholder(tf.float32, [None, sh])
 y = tf.placeholder(tf.float32, [None, 1])
 p5 = tf.constant(0.5)
 W = tf.Variable(tf.random_normal([sh, 1], mean=0.0, stddev=0.05))
b = tf.Variable([0.])
 y_pred = tf.matmul(x, W)+ b
y_pred_sigmoid = tf.sigmoid(y_pred)
 x_entropy = tf.nn.sigmoid_cross_entropy_with_logits(logits=y_pred, labels=y)
 loss = tf.reduce_mean(x_entropy)
 train_step = tf.train.GradientDescentOptimizer(alpha ).minimize(loss)
 delta = tf.abs(( y - y_pred_sigmoid))
correct_prediction = tf.cast(tf.less(delta, p5), tf.int32)
 prediction = tf.cast(tf.round(y_pred_sigmoid), tf.int32)
 accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
init = tf.initialize_all_variables()
costs = []
accs = []
with tf.Session() as sess:
sess.run(init)
 print('Training...')
 for i in range(nEpoch):
fd_train = {x: X, y: Y.reshape(( -1, 1)) }
train_step.run(fd_train)
 fd_test = {x: Xtest}
pred = prediction.eval(fd_test)
 if i % 5000 == 0:
loss_step = loss.eval(fd_train)
 train_accuracy = accuracy.eval(fd_train)
 costs.append(loss_step)
 accs.append(train_accuracy)
 print("cost at epoch ", i, " is: ", loss_step)
 return costs, pred, accs
find_w(year_list,
np.concatenate([np.zeros(1), np.logspace(-1, 1, 10)]),
np.logspace(-3, 3, 21))<find_best_params>
def neuralNetwork(X, Y, Xtest, alpha, nEpoch, sh):
tf.reset_default_graph()
x = tf.placeholder(tf.float32, [sh, None])
 y = tf.placeholder(tf.float32, [1, None])
 p5 = tf.constant(0.5)
 W1 = tf.get_variable("W1", [5, sh], initializer = tf.contrib.layers.xavier_initializer())
 b1 = tf.get_variable("b1", [5, 1], initializer = tf.zeros_initializer())
 W2 = tf.get_variable("W2", [2, 5], initializer = tf.contrib.layers.xavier_initializer())
 b2 = tf.get_variable("b2", [2, 1], initializer = tf.zeros_initializer())
 W3 = tf.get_variable("W3", [4, 2], initializer = tf.contrib.layers.xavier_initializer())
 b3 = tf.get_variable("b3", [4, 1], initializer = tf.zeros_initializer())
 W4 = tf.get_variable("W4", [1, 4], initializer = tf.contrib.layers.xavier_initializer())
 b4 = tf.get_variable("b4", [1, 1], initializer = tf.zeros_initializer())
 Z1 = tf.add(tf.matmul(W1, x), b1)
 A1 = tf.nn.relu(Z1)
 Z2 = tf.add(tf.matmul(W2, A1), b2)
 A2 = tf.nn.relu(Z2)
 Z3 = tf.add(tf.matmul(W3, A2), b3)
 A3 = tf.nn.relu(Z3)
 Z4 = tf.add(tf.matmul(W4, A3), b4)
 logits = tf.transpose(Z4)
 labels = tf.transpose(y)
 y_pred_sigmoid = tf.sigmoid(logits)
 x_entropy = tf.nn.sigmoid_cross_entropy_with_logits(logits=logits, labels=labels)
 loss = tf.reduce_mean(x_entropy)
 train_step = tf.train.AdamOptimizer(alpha ).minimize(loss)
 delta = tf.abs(( y - y_pred_sigmoid))
correct_prediction = tf.cast(tf.less(delta, p5), tf.int32)
 prediction = tf.cast(tf.round(y_pred_sigmoid), tf.int32)
 accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
init = tf.global_variables_initializer()
costs = []
accs = []
with tf.Session() as sess:
sess.run(init)
 print('Training...')
 for i in range(nEpoch):
fd_train = {x: X, y: Y.reshape(( 1, -1)) }
train_step.run(fd_train)
 fd_test = {x: Xtest}
pred = prediction.eval(fd_test)
 if i % 5000 == 0:
loss_step = loss.eval(fd_train)
 train_accuracy = accuracy.eval(fd_train)
 costs.append(loss_step)
 accs.append(train_accuracy)
 print("Cost at epoch ", i, " is ", loss_step)
 return costs, pred, accs
find_w(year_list,
np.linspace(1, 4, 7),
np.logspace(-3, 3, 21))<find_best_params>
m = trainData['Name'].count()
X = trainData[['Pclass', 'Sex', 'Age', 'SibSp', 'Parch']]
X['Sex'] = X['Sex'].replace(to_replace=['male', 'female'], value=[0, 1])
 Y = trainData['Survived']
Xtest = testData[['Pclass', 'Sex', 'Age', 'SibSp', 'Parch']]
Xidx = testData[['PassengerId']]
Xtest['Sex'] = Xtest['Sex'].replace(to_replace=['male', 'female'], value=[0, 1] )
find_c(year_list, 2.5, np.linspace(2, 8, 31))<predict_on_test>
X_age = X[np.isnan(X['Age'])!=True]
Y_age = Y[np.isnan(X['Age'])!=True]
m = len(Y_age)
 Y_age = np.reshape(Y_age,(m, 1))
Xtest_age = Xtest[np.isnan(Xtest['Age'])!=True]
Xidx_age = Xidx[np.isnan(Xtest['Age'])!=True]
costs, predictions, accuracy = logisticRegression(X_age, Y_age, Xtest_age, 0.01, 100000, 5)
 print("Model trained!" )
def predict(w, c, filename):
year = 2018
sr = season_result[season_result.Season == year]
x_sr = encode_team(sr)
 y_sr = np.ones(x_sr.shape[0], dtype=np.int64)
 x_train = sp.sparse.vstack([x_sr, -x_sr])
 y_train = np.concatenate([y_sr, -y_sr])
 log_ratio = np.concatenate([sr.log_ratio, sr.log_ratio])
 x_test = encode_team(sample_submission)
 if w is None:
w_train = None
else:
w_train = w * log_ratio.min() + log_ratio
cls = LogisticRegression(C=c)
 cls.fit(x_train, y_train, w_train)
 col = list(cls.classes_ ).index(1)
 y_pred = cls.predict_proba(x_test)[:, col]
submission = pd.DataFrame()
submission['ID'] = sample_submission.ID
submission['Pred'] = y_pred
submission.to_csv(filename, index=False)
 return cls<predict_on_test>
X_noage = X[['Pclass', 'Sex', 'SibSp', 'Parch']]
Y_noage = Y
Xtest_noage = Xtest[np.isnan(Xtest['Age'])]
Xtest_noage = Xtest_noage[['Pclass', 'Sex', 'SibSp', 'Parch']]
Xidx_noage = Xidx[np.isnan(Xtest['Age'])]
costs_na, predictions_na, accuracy_na = logisticRegression(X_noage, Y_noage, Xtest_noage, 0.01, 20000, 4)
 print("Model trained!" )
lr = predict(w=None, c=1.1, filename='predict_with_logistic.csv')
 wlr = predict(w=2.5, c=3.6, filename='predict_with_weighted_logistic.csv' )<load_from_csv>
X_age = X[np.isnan(X['Age'])!=True]
Y_age = Y[np.isnan(X['Age'])!=True]
m = len(Y_age)
 Y_age = np.reshape(Y_age,(m, 1))
Xtest_age = Xtest[np.isnan(Xtest['Age'])!=True]
Xidx_age = Xidx[np.isnan(Xtest['Age'])!=True]
costs_nn, predictions_nn, accuracy_nn = neuralNetwork(np.transpose(X_age), np.transpose(Y_age), np.transpose(Xtest_age), 0.003, 50000, 5)
 print("Model trained!" )
df_seeds = pd.read_csv('.. /input/NCAATourneySeeds.csv')
 df_tour = pd.read_csv('.. /input/NCAATourneyCompactResults.csv' )<merge>
X_noage = X[['Pclass', 'Sex', 'SibSp', 'Parch']]
Y_noage = Y
Xtest_noage = Xtest[np.isnan(Xtest['Age'])]
Xtest_noage = Xtest_noage[['Pclass', 'Sex', 'SibSp', 'Parch']]
Xidx_noage = Xidx[np.isnan(Xtest['Age'])]
costs_na_nn, predictions_na_nn, accuracy_na_nn = neuralNetwork(np.transpose(X_noage), np.transpose(Y_noage), np.transpose(Xtest_noage), 0.003, 20000, 4)
 print("Model trained!" )
df_seeds['seed_int'] = df_seeds['Seed'].apply(lambda x : int(x[1:3]))
df_winseeds = df_seeds.loc[:, ['TeamID', 'Season', 'seed_int']].rename(columns={'TeamID':'WTeamID', 'seed_int':'WSeed'})
 df_lossseeds = df_seeds.loc[:, ['TeamID', 'Season', 'seed_int']].rename(columns={'TeamID':'LTeamID', 'seed_int':'LSeed'})
 df_dummy = pd.merge(left=df_tour, right=df_winseeds, how='left', on=['Season', 'WTeamID'])
 df_concat = pd.merge(left=df_dummy, right=df_lossseeds, on=['Season', 'LTeamID'] )<feature_engineering>
pd.options.mode.chained_assignment = None
fare_avg = np.sum(trainData['Fare'])/trainData['Fare'].count()
age_avg = np.sum(trainData['Age'])/trainData['Age'].count()
X2 = trainData[['Pclass', "Age", "Sex", "SibSp", "Parch", 'Fare', 'Embarked']]
X2['Sex'] = X2['Sex'].replace(to_replace=['male', 'female'], value=[0, 1])
 X2['Embarked'] = X2['Embarked'].replace(to_replace=['S', 'C', 'Q'], value=[0, 1, 2])
 for i in range(0, 891):
if np.isnan(X2['Fare'][i]):
X2['Fare'][i] = fare_avg
if np.isnan(X2['Age'][i]):
X2['Age'][i] = age_avg
if np.isnan(X2['Embarked'][i]):
X2['Embarked'][i] = 0
X2test = testData[['Pclass', "Age", "Sex", "SibSp", "Parch", 'Fare', 'Embarked']]
X2test['Sex'] = X2test['Sex'].replace(to_replace=['male', 'female'], value=[0, 1])
 X2test['Embarked'] = X2test['Embarked'].replace(to_replace=['S', 'C', 'Q'], value=[0, 1, 2])
 for i in range(0, 418):
if np.isnan(X2test['Fare'][i]):
X2test['Fare'][i] = fare_avg
if np.isnan(X2test['Age'][i]):
X2test['Age'][i] = age_avg
if np.isnan(X2test['Embarked'][i]):
X2test['Embarked'][i] = 0
costs_agelg, predictions_agelg, accuracy_agelg = logisticRegression(X2, Y, X2test, 0.003, 40000, 7)
 print("Logistic regression trained")
 costs_agenn, predictions_agenn, accuracy_agenn = neuralNetwork(np.transpose(X2), np.transpose(Y), np.transpose(X2test), 0.003, 100000, 7)
 print("Neural network has been trained")
 X3 = trainData[['Pclass', "Age", "Sex", "SibSp", "Parch", 'Fare', 'Embarked']]
X3['Sex'] = X3['Sex'].replace(to_replace=['male', 'female'], value=[0, 1])
 X3['Embarked'] = X3['Embarked'].replace(to_replace=['S', 'C', 'Q'], value=[0, 1, 2])
 imp = Imputer()
X3 = imp.fit_transform(X3)
 X3test = testData[['Pclass', "Age", "Sex", "SibSp", "Parch", 'Fare', 'Embarked']]
X3test['Sex'] = X3test['Sex'].replace(to_replace=['male', 'female'], value=[0, 1])
 X3test['Embarked'] = X3test['Embarked'].replace(to_replace=['S', 'C', 'Q'], value=[0, 1, 2])
 X3test = imp.fit_transform(X3test)
 costs_imp, predictions_imp, accuracy_imp = neuralNetwork(np.transpose(X3), np.transpose(Y), np.transpose(X3test), 0.003, 100000, 7)
 print("Neural network with imputation is trained!" )
<load_from_csv><EOS> 
names = testData[['PassengerId']]
result_age = np.concatenate(( np.reshape(Xidx_age,(332, 1)) ,predictions), axis=1)
 result_noage = np.concatenate(( np.reshape(Xidx_noage,(86, 1)) ,predictions_na), axis=1)
 result = np.concatenate(( result_age, result_noage), axis=0)
 myfile = open('output_lg.csv','w')
 column= ['PassengerId', 'Survived']
wrtr = csv.writer(myfile, delimiter=',')
 wrtr.writerow(column)
 wrtr.writerows(result)
 myfile.close()
result_age = np.concatenate(( np.reshape(Xidx_age,(332, 1)) ,predictions_nn), axis=1)
 result_noage = np.concatenate(( np.reshape(Xidx_noage,(86, 1)) ,predictions_na_nn), axis=1)
 result_nn = np.concatenate(( result_age, result_noage), axis=0)
 myfile = open('output_nn.csv','w')
 column= ['PassengerId', 'Survived']
wrtr = csv.writer(myfile, delimiter=',')
 wrtr.writerow(column)
 wrtr.writerows(result_nn)
 myfile.close()
result_agelg = np.concatenate(( np.reshape(Xidx,(418, 1)) ,predictions_agelg), axis=1)
 myfile = open('output_agelg.csv','w')
 column= ['PassengerId', 'Survived']
wrtr = csv.writer(myfile, delimiter=',')
 wrtr.writerow(column)
 wrtr.writerows(result_agelg)
 myfile.close()
result_agenn = np.concatenate(( np.reshape(Xidx,(418, 1)) ,predictions_agenn), axis=1)
 myfile = open('output_agenn.csv','w')
 column= ['PassengerId', 'Survived']
wrtr = csv.writer(myfile, delimiter=',')
 wrtr.writerow(column)
 wrtr.writerows(result_agenn)
 myfile.close()
result_imp = np.concatenate(( np.reshape(Xidx,(418, 1)) ,predictions_imp), axis=1)
 myfile = open('output_imp.csv','w')
 column= ['PassengerId', 'Survived']
wrtr = csv.writer(myfile, delimiter=',')
 wrtr.writerow(column)
 wrtr.writerows(result_imp)
 myfile.close()