Kaggle competition: Spooky Author Identification
<SOS> metric: MulticlassLoss Kaggle data source: spooky-author-identification<choose_model_class>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import nltk
import string
model_name = "LaNet5"
model = Sequential()
model.add(Reshape(target_shape=(1, 28, 28), input_shape=(784,)))
 model.add(Conv2D(kernel_size=(3, 3), filters=6, padding="same", data_format="channels_first", kernel_initializer="uniform", use_bias=False))
model.add(MaxPooling2D(pool_size=(2, 2), data_format="channels_first"))
model.add(Conv2D(kernel_size=(5, 5), filters=16, padding="same", data_format="channels_first", kernel_initializer="uniform", use_bias=False))
model.add(MaxPooling2D(pool_size=(2, 2), data_format="channels_first"))
model.add(Conv2D(kernel_size=(5, 5), filters=120, padding="same", data_format="channels_first", kernel_initializer="uniform", use_bias=False))
model.add(Flatten())
 model.add(Dense(output_dim=120, activation='relu'))
model.add(Dense(output_dim=120, activation='relu'))
model.add(Dense(output_dim=10, activation='softmax'))
adam = keras.optimizers.Adam(lr=0.0005, beta_1=0.9, beta_2=0.999, epsilon=1e-08)
 model.compile(loss='categorical_crossentropy', optimizer=adam, metrics=['accuracy'])
 model.fit(x_train, y_train, epochs=30, batch_size=64)
 y_pred = model.predict_classes(x_test)
 output_prediction(y_pred, model_name)
 y_all_pred[0] = y_pred<choose_model_class>
df_train = pd.read_csv('.. /input/train.csv')
 df_test = pd.read_csv('.. /input/test.csv' )
model_name = "CNN"
model = Sequential()
model.add(Reshape(target_shape=(1, 28, 28), input_shape=(784,)))
 model.add(Conv2D(kernel_size=(3, 3), filters=32, padding="same", data_format="channels_first", kernel_initializer="uniform", use_bias=False))
model.add(AveragePooling2D(pool_size=(2, 2), data_format="channels_first"))
model.add(Conv2D(kernel_size=(3, 3), filters=32, padding="same", data_format="channels_first", kernel_initializer="uniform", use_bias=False))
model.add(AveragePooling2D(pool_size=(2, 2), data_format="channels_first"))
model.add(Conv2D(kernel_size=(3, 3), filters=64, padding="same", data_format="channels_first", kernel_initializer="uniform", use_bias=False))
model.add(AveragePooling2D(pool_size=(2, 2), data_format="channels_first"))
model.add(Dropout(0.25))
model.add(Flatten())
 model.add(Dense(output_dim=256, activation='relu'))
model.add(Dropout(0.25))
model.add(Dense(output_dim=256, activation='relu'))
model.add(Dense(output_dim=10, activation='softmax'))
adam = keras.optimizers.Adam(lr=0.0005, beta_1=0.9, beta_2=0.999, epsilon=1e-08)
 model.compile(loss='categorical_crossentropy', optimizer=adam, metrics=['accuracy'])
 model.fit(x_train, y_train, epochs=50, batch_size=128)
 y_pred = model.predict_classes(x_test)
 output_prediction(y_pred, model_name)
 y_all_pred[1] = y_pred<choose_model_class>
def remove_punctuation(text):
for punct in string.punctuation:
text = text.replace(punct,'')
 return text
df_train.text = df_train.text.apply(remove_punctuation)
 df_test.text = df_test.text.apply(remove_punctuation )
model_name = "ComplexCNN"
model = Sequential()
model.add(Reshape(target_shape=(1, 28, 28), input_shape=(784,)))
 model.add(Conv2D(32,(3, 3), padding='same', activation='relu'))
model.add(Conv2D(32,(3, 3), activation='relu', padding='same'))
model.add(MaxPooling2D(pool_size=(2, 2), data_format="channels_first"))
model.add(Conv2D(32,(3, 3), padding='same', activation='relu'))
model.add(Conv2D(32,(3, 3), padding='same', activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2), data_format="channels_first"))
model.add(Conv2D(64,(3, 3), padding='same', activation='relu'))
model.add(Conv2D(64,(3, 3), padding='same', activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2), data_format="channels_first"))
model.add(Dropout(0.25))
model.add(Flatten())
 model.add(Dense(output_dim=256, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(output_dim=256, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(output_dim=10, activation='softmax'))
adam = keras.optimizers.Adam(lr=0.0005, beta_1=0.9, beta_2=0.999, epsilon=1e-08)
 model.compile(loss='categorical_crossentropy', optimizer=adam, metrics=['accuracy'])
 model.fit(x_train, y_train, epochs=80, batch_size=128)
 y_pred = model.predict_classes(x_test)
 output_prediction(y_pred, model_name)
 y_all_pred[2] = y_pred<data_type_conversions>
APPLY_STEMMING = False
if APPLY_STEMMING:
stemmer = stm.PorterStemmer()
df_train.text = df_train.text.apply(lambda text: " ".join([stemmer.stem(word)for word in text.split(" ")]))
df_test.text = df_test.text.apply(lambda text: " ".join([stemmer.stem(word)for word in text.split(" ")]))
model_name = "Ensemble"
print(y_pred.shape)
 y_ensem_pred = np.zeros(( n_samples_test,))
for i,line in enumerate(y_all_pred.T):
y_ensem_pred[i] = np.argmax(np.bincount(line))
print(y_ensem_pred.shape, y_ensem_pred)
 y_ensem_pred = y_ensem_pred.astype("int64")
 output_prediction(y_ensem_pred, model_name )<load_from_csv>
count_vect = CountVectorizer(ngram_range=(1,3),stop_words='english')
 count_vectorized = count_vect.fit_transform(df_train.text )
trainData = pd.read_csv('.. /input/train.csv')
 testData = pd.read_csv('.. /input/test.csv')
 trainData.head(5 )<count_values>
tfidf_vect = TfidfVectorizer(ngram_range=(1,3), stop_words='english')
 tfidf_vectorized = tfidf_vect.fit_transform(df_train.text )
print("Number of non-null values in each column in dataset")
 print("Survived = ", trainData['Survived'].count())
 print("Pclass = ", trainData['Pclass'].count())
 print("Sex = ", trainData['Sex'].count())
 print("Age = ", trainData['Age'].count())
 print("SibSp = ", trainData['SibSp'].count())
 print("Parch = ", trainData['Parch'].count())
 print("Ticket = ", trainData['Ticket'].count())
 print("Fare = ", trainData['Fare'].count())
 print("Cabin = ", trainData['Cabin'].count())
 print("Embarked = ", trainData['Embarked'].count() )<compute_train_metric>
X_train_count, X_test_count, y_train_count, y_test_count = train_test_split(count_vectorized, df_train.author, train_size=0.75)
 X_train_tfidf, X_test_tfidf, y_train_tfidf, y_test_tfidf = train_test_split(tfidf_vectorized, df_train.author, train_size=0.75 )
def logisticRegression(X, Y, Xtest, alpha, nEpoch, sh, lamb=0):
x = tf.placeholder(tf.float32, [None, sh])
 y = tf.placeholder(tf.float32, [None, 1])
 p5 = tf.constant(0.5)
 W = tf.Variable(tf.random_normal([sh, 1], mean=0.0, stddev=0.05))
b = tf.Variable([0.])
 y_pred = tf.matmul(x, W)+ b
y_pred_sigmoid = tf.sigmoid(y_pred)
 x_entropy = tf.nn.sigmoid_cross_entropy_with_logits(logits=y_pred, labels=y)
 loss = tf.reduce_mean(x_entropy)
 train_step = tf.train.GradientDescentOptimizer(alpha ).minimize(loss)
 delta = tf.abs(( y - y_pred_sigmoid))
correct_prediction = tf.cast(tf.less(delta, p5), tf.int32)
 prediction = tf.cast(tf.round(y_pred_sigmoid), tf.int32)
 accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
init = tf.initialize_all_variables()
costs = []
accs = []
with tf.Session() as sess:
sess.run(init)
 print('Training...')
 for i in range(nEpoch):
fd_train = {x: X, y: Y.reshape(( -1, 1)) }
train_step.run(fd_train)
 fd_test = {x: Xtest}
pred = prediction.eval(fd_test)
 if i % 5000 == 0:
loss_step = loss.eval(fd_train)
 train_accuracy = accuracy.eval(fd_train)
 costs.append(loss_step)
 accs.append(train_accuracy)
 print("cost at epoch ", i, " is: ", loss_step)
 return costs, pred, accs<prepare_x_and_y>
model_count_NB = MultinomialNB()
model_tfidf_NB = MultinomialNB()
model_count_NB.fit(X_train_count, y_train_count)
 model_tfidf_NB.fit(X_train_tfidf, y_train_tfidf )
def neuralNetwork(X, Y, Xtest, alpha, nEpoch, sh):
tf.reset_default_graph()
x = tf.placeholder(tf.float32, [sh, None])
 y = tf.placeholder(tf.float32, [1, None])
 p5 = tf.constant(0.5)
 W1 = tf.get_variable("W1", [5, sh], initializer = tf.contrib.layers.xavier_initializer())
 b1 = tf.get_variable("b1", [5, 1], initializer = tf.zeros_initializer())
 W2 = tf.get_variable("W2", [2, 5], initializer = tf.contrib.layers.xavier_initializer())
 b2 = tf.get_variable("b2", [2, 1], initializer = tf.zeros_initializer())
 W3 = tf.get_variable("W3", [4, 2], initializer = tf.contrib.layers.xavier_initializer())
 b3 = tf.get_variable("b3", [4, 1], initializer = tf.zeros_initializer())
 W4 = tf.get_variable("W4", [1, 4], initializer = tf.contrib.layers.xavier_initializer())
 b4 = tf.get_variable("b4", [1, 1], initializer = tf.zeros_initializer())
 Z1 = tf.add(tf.matmul(W1, x), b1)
 A1 = tf.nn.relu(Z1)
 Z2 = tf.add(tf.matmul(W2, A1), b2)
 A2 = tf.nn.relu(Z2)
 Z3 = tf.add(tf.matmul(W3, A2), b3)
 A3 = tf.nn.relu(Z3)
 Z4 = tf.add(tf.matmul(W4, A3), b4)
 logits = tf.transpose(Z4)
 labels = tf.transpose(y)
 y_pred_sigmoid = tf.sigmoid(logits)
 x_entropy = tf.nn.sigmoid_cross_entropy_with_logits(logits=logits, labels=labels)
 loss = tf.reduce_mean(x_entropy)
 train_step = tf.train.AdamOptimizer(alpha ).minimize(loss)
 delta = tf.abs(( y - y_pred_sigmoid))
correct_prediction = tf.cast(tf.less(delta, p5), tf.int32)
 prediction = tf.cast(tf.round(y_pred_sigmoid), tf.int32)
 accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
init = tf.global_variables_initializer()
costs = []
accs = []
with tf.Session() as sess:
sess.run(init)
 print('Training...')
 for i in range(nEpoch):
fd_train = {x: X, y: Y.reshape(( 1, -1)) }
train_step.run(fd_train)
 fd_test = {x: Xtest}
pred = prediction.eval(fd_test)
 if i % 5000 == 0:
loss_step = loss.eval(fd_train)
 train_accuracy = accuracy.eval(fd_train)
 costs.append(loss_step)
 accs.append(train_accuracy)
 print("Cost at epoch ", i, " is ", loss_step)
 return costs, pred, accs<prepare_x_and_y>
predictions_count = model_count_NB.predict(X_test_count)
 predictions_tfidf = model_tfidf_NB.predict(X_test_tfidf )
m = trainData['Name'].count()
X = trainData[['Pclass', 'Sex', 'Age', 'SibSp', 'Parch']]
X['Sex'] = X['Sex'].replace(to_replace=['male', 'female'], value=[0, 1])
 Y = trainData['Survived']
Xtest = testData[['Pclass', 'Sex', 'Age', 'SibSp', 'Parch']]
Xidx = testData[['PassengerId']]
Xtest['Sex'] = Xtest['Sex'].replace(to_replace=['male', 'female'], value=[0, 1] )<train_model>
accuracy_count = accuracy_score(y_test_count, predictions_count)
 accuracy_tfidf = accuracy_score(y_test_tfidf, predictions_tfidf)
 print('Count Vectorized Words Accuracy:', accuracy_count)
 print('TfIdf Vectorized Words Accuracy:', accuracy_tfidf )
X_age = X[np.isnan(X['Age'])!=True]
Y_age = Y[np.isnan(X['Age'])!=True]
m = len(Y_age)
 Y_age = np.reshape(Y_age,(m, 1))
Xtest_age = Xtest[np.isnan(Xtest['Age'])!=True]
Xidx_age = Xidx[np.isnan(Xtest['Age'])!=True]
costs, predictions, accuracy = logisticRegression(X_age, Y_age, Xtest_age, 0.01, 100000, 5)
 print("Model trained!" )<train_model>
model_NB = MultinomialNB()
model_NB.fit(count_vectorized, df_train.author )
X_noage = X[['Pclass', 'Sex', 'SibSp', 'Parch']]
Y_noage = Y
Xtest_noage = Xtest[np.isnan(Xtest['Age'])]
Xtest_noage = Xtest_noage[['Pclass', 'Sex', 'SibSp', 'Parch']]
Xidx_noage = Xidx[np.isnan(Xtest['Age'])]
costs_na, predictions_na, accuracy_na = logisticRegression(X_noage, Y_noage, Xtest_noage, 0.01, 20000, 4)
 print("Model trained!" )<features_selection>
X_test = count_vect.transform(df_test.text )
X_age = X[np.isnan(X['Age'])!=True]
Y_age = Y[np.isnan(X['Age'])!=True]
m = len(Y_age)
 Y_age = np.reshape(Y_age,(m, 1))
Xtest_age = Xtest[np.isnan(Xtest['Age'])!=True]
Xidx_age = Xidx[np.isnan(Xtest['Age'])!=True]
costs_nn, predictions_nn, accuracy_nn = neuralNetwork(np.transpose(X_age), np.transpose(Y_age), np.transpose(Xtest_age), 0.003, 50000, 5)
 print("Model trained!" )<prepare_x_and_y>
predicted_values = model_NB.predict_proba(X_test )
X_noage = X[['Pclass', 'Sex', 'SibSp', 'Parch']]
Y_noage = Y
Xtest_noage = Xtest[np.isnan(Xtest['Age'])]
Xtest_noage = Xtest_noage[['Pclass', 'Sex', 'SibSp', 'Parch']]
Xidx_noage = Xidx[np.isnan(Xtest['Age'])]
costs_na_nn, predictions_na_nn, accuracy_na_nn = neuralNetwork(np.transpose(X_noage), np.transpose(Y_noage), np.transpose(Xtest_noage), 0.003, 20000, 4)
 print("Model trained!" )<categorify>
model_NB.classes_
<save_to_csv><EOS> 
df_submission = pd.DataFrame({
'id': df_test.id.values,
'EAP': predicted_values[:,0],
'HPL': predicted_values[:,1],
'MWS': predicted_values[:,2]
})
 datetime = time.strftime("%Y%m%d_%H%M%S")
 filename = 'submission_' + datetime + '_acc_' + str(accuracy_count)+ '.csv'
df_submission.to_csv(filename, index=False)
 print('File',filename,'created.' )