Kaggle competition: Spooky Author Identification
<SOS> metric: MulticlassLoss Kaggle data source: spooky-author-identification<define_variables>
alpha_tokenizer = RegexpTokenizer('[A-Za-z]\w+')
 lemmatizer = WordNetLemmatizer()
stop = stopwords.words('english' )
epochs = 30
batch_size = 86<init_hyperparams>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 test_id = test['id'].values
author_mapping = {'EAP':0, 'HPL':1, 'MWS':2}
y_train = train['author'].map(author_mapping ).values
datagen = ImageDataGenerator(
 featurewise_center = False,
samplewise_center = False,
featurewise_std_normalization = False,
samplewise_std_normalization = False,
zca_whitening = False,
rotation_range = 10,
zoom_range = 0.1,
width_shift_range = 0.1,
height_shift_range = 0.1,
horizontal_flip = False,
vertical_flip = False)
 datagen.fit(X_train )<train_model>
vectorizers = [ 
('TF-IDF + SVD', Pipeline([('tfidf', TfidfVectorizer(ngram_range=(1, 3), analyzer='word', binary=False)) , 
('svd', TruncatedSVD(n_components=150)) ,
])) , 
('TF-IDF + SVD + Normalizer', Pipeline([('tfidf', TfidfVectorizer(ngram_range=(1, 3), analyzer='word', binary=False)) , 
('svd', TruncatedSVD(n_components=150)) , 
('norm', Normalizer()),
]))
]
history = model.fit_generator(datagen.flow(X_train, Y_train, batch_size = batch_size), epochs = epochs, validation_data =(X_val, Y_val),
verbose = 2, steps_per_epoch = X_train.shape[0] // batch_size, callbacks = [learning_rate_reduction] )<predict_on_test>
estimators = [ 
(KNeighborsClassifier(n_neighbors=3), 'K-Nearest Neighbors', 'yellow'), 
(SVC(C=1, cache_size=200, class_weight=None, coef0=0.0, decision_function_shape='ovr', degree=3, gamma='auto', kernel='linear', max_iter=-1, probability=False, random_state=None, shrinking=True,tol=0.001, verbose=False), 'Support Vector Machine', 'red'), 
(LogisticRegression(tol=1e-8, penalty='l2', C=0.1), 'Logistic Regression', 'green'), 
(MultinomialNB() , 'Naive Bayes', 'magenta'), 
(RandomForestClassifier(n_estimators=10, criterion='gini'), 'Random Forest', 'gray'), 
(None, 'XGBoost', 'pink')
 ]
results = model.predict(test)
 results = np.argmax(results,axis = 1)
 results = pd.Series(results,name="Label" )<save_to_csv>
params = {}
params['objective'] = 'multi:softprob'
params['eta'] = 0.1
params['max_depth'] = 3
params['silent'] = 1
params['num_class'] = 3
params['eval_metric'] = 'mlogloss'
params['min_child_weight'] = 1
params['subsample'] = 0.8
params['colsample_bytree'] = 0.3
params['seed'] = 0
submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),results],axis = 1)
 submission.to_csv("submission.csv",index=False )<import_modules>
def vectorize() :
test_size = 0.3
train_split, test_split = train_test_split(train, test_size=test_size)
 y_train_split = train_split['author'].map(author_mapping ).values
y_test_split = test_split['author'].map(author_mapping ).values
for vectorizer in vectorizers:
print(vectorizer[0] + '
')
 X = vectorizer[1].fit_transform(train.text.values)
 X_train, X_test = train_test_split(X, test_size=test_size)
 for estimator in estimators:
if estimator[1] == 'XGBoost':
xgtrain = xgb.DMatrix(X_train, y_train_split)
 xgtest = xgb.DMatrix(X_test)
 model = xgb.train(params=list(params.items()), dtrain=xgtrain, num_boost_round=40)
 predictions = model.predict(xgtest, ntree_limit=model.best_ntree_limit ).argmax(axis=1)
 else:
estimator[0].fit(X_train, y_train_split)
 predictions = estimator[0].predict(X_test)
 print(accuracy_score(predictions, y_test_split), estimator[1] )
import numpy as np
import pandas as pd
import os
import time
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
import seaborn as sb
import itertools
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix
from keras.utils.np_utils import to_categorical
from keras.models import Sequential
from keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPool2D
from keras.layers.advanced_activations import LeakyReLU
from keras.optimizers import RMSprop, Adam
from keras.preprocessing.image import ImageDataGenerator 
<load_from_csv>
train['num_words'] = train.text.apply(lambda x: len(str(x ).split()))
test['num_words'] = test.text.apply(lambda x: len(str(x ).split()))
train['num_unique_words'] = train.text.apply(lambda x: len(set(str(x ).split())))
 test['num_unique_words'] = test.text.apply(lambda x: len(set(str(x ).split())))
 train['num_chars'] = train.text.apply(lambda x: len(str(x)))
 test['num_chars'] = test.text.apply(lambda x: len(str(x)))
 train['num_stopwords'] = train.text.apply(lambda x: len([w for w in str(x ).lower().split() if w in stop]))
test['num_stopwords'] = test.text.apply(lambda x: len([w for w in str(x ).lower().split() if w in stop]))
train['mean_word_len'] = train.apply(lambda x: np.mean([len(w)for w in str(x ).split() ]))
test['mean_word_len'] = test.apply(lambda x: np.mean([len(w)for w in str(x ).split() ]))
os.listdir('.. /input')
 train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )<count_values>
train_text = [' '.join([lemmatizer.lemmatize(word.lower())for word in alpha_tokenizer.tokenize(sent)if word.lower() not in stop])for sent in train.text.values]
test_text = [' '.join([lemmatizer.lemmatize(word.lower())for word in alpha_tokenizer.tokenize(sent)if word.lower() not in stop])for sent in test.text.values]
print(X_train.shape)
 print(test.shape)
 sb.countplot(Y_train)
 Y_train.value_counts()<train_model>
vectorizer = CountVectorizer(ngram_range=(1,7), analyzer='char')
 full = vectorizer.fit_transform(train_text + test_text)
 X_train = vectorizer.transform(train_text)
 X_test = vectorizer.transform(test_text)
 pred_full_test = 0
pred_train = np.zeros([train.shape[0], 3])
 for dev_index, val_index in KFold(n_splits=5, shuffle=True, random_state=42 ).split(train.drop(['id', 'author'], axis=1)) :
dev_X, val_X = X_train[dev_index], X_train[val_index]
dev_y, val_y = y_train[dev_index], y_train[val_index]
model = MultinomialNB()
model.fit(dev_X, dev_y)
 pred_full_test = pred_full_test + model.predict_proba(X_test)
 pred_train[val_index,:] = model.predict_proba(val_X)
 pred_full_test = pred_full_test / 5.
 train['CH_EAP'] = pred_train[:,0]
train['CH_HPL'] = pred_train[:,1]
train['CH_MWS'] = pred_train[:,2]
test['CH_EAP'] = pred_full_test[:,0]
test['CH_HPL'] = pred_full_test[:,1]
test['CH_MWS'] = pred_full_test[:,2]
print("Previous shape of Y_train:")
 print(Y_train.shape)
 Y_train = to_categorical(Y_train, num_classes = 10)
 print("New shape of Y_train:")
 print(Y_train.shape )<split>
vectorizer = CountVectorizer(stop_words='english', ngram_range=(1,3))
full = vectorizer.fit_transform(train_text + test_text)
 X_train = vectorizer.transform(train_text)
 X_test = vectorizer.transform(test_text)
 pred_full_test = 0
pred_train = np.zeros([train.shape[0], 3])
 for dev_index, val_index in KFold(n_splits=5, shuffle=True, random_state=42 ).split(train.drop(['id', 'author'], axis=1)) :
dev_X, val_X = X_train[dev_index], X_train[val_index]
dev_y, val_y = y_train[dev_index], y_train[val_index]
model = MultinomialNB()
model.fit(dev_X, dev_y)
 pred_full_test = pred_full_test + model.predict_proba(X_test)
 pred_train[val_index,:] = model.predict_proba(val_X)
 pred_full_test = pred_full_test / 5.
 train['C_EAP'] = pred_train[:,0]
train['C_HPL'] = pred_train[:,1]
train['C_MWS'] = pred_train[:,2]
test['C_EAP'] = pred_full_test[:,0]
test['C_HPL'] = pred_full_test[:,1]
test['C_MWS'] = pred_full_test[:,2]
X_train, X_dev, Y_train, Y_dev = train_test_split(X_train, Y_train, test_size = 0.1 )<define_variables>
vectorizer = TfidfVectorizer(ngram_range=(1,5), analyzer='char')
 full = vectorizer.fit_transform(train_text + test_text)
 X_train = vectorizer.transform(train_text)
 X_test = vectorizer.transform(test_text)
 pred_full_test = 0
pred_train = np.zeros([train.shape[0], 3])
 for dev_index, val_index in KFold(n_splits=5, shuffle=True, random_state=42 ).split(train.drop(['id', 'author'], axis=1)) :
dev_X, val_X = X_train[dev_index], X_train[val_index]
dev_y, val_y = y_train[dev_index], y_train[val_index]
model = MultinomialNB()
model.fit(dev_X, dev_y)
 pred_full_test = pred_full_test + model.predict_proba(X_test)
 pred_train[val_index,:] = model.predict_proba(val_X)
 pred_full_test = pred_full_test / 5.
 train['T_EAP'] = pred_train[:,0]
train['T_HPL'] = pred_train[:,1]
train['T_MWS'] = pred_train[:,2]
test['T_EAP'] = pred_full_test[:,0]
test['T_HPL'] = pred_full_test[:,1]
test['T_MWS'] = pred_full_test[:,2]
datagen = ImageDataGenerator(
 rotation_range=10,
zoom_range = 0.10,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False)
<choose_model_class>
svd = TruncatedSVD(n_components=20, algorithm='arpack')
 svd.fit(full)
 train_svd = pd.DataFrame(svd.transform(X_train))
test_svd = pd.DataFrame(svd.transform(X_test))
train_svd.columns = ['SVD_' + str(i)for i in range(20)]
test_svd.columns = ['SVD_' + str(i)for i in range(20)]
train = pd.concat([train, train_svd], axis=1)
 test = pd.concat([test, test_svd], axis=1 )
model = Sequential()
model.add(Conv2D(filters=64, kernel_size=(3,3), padding="same", input_shape=(28, 28, 1)))
 model.add(LeakyReLU(0.1))
model.add(Conv2D(filters=64, kernel_size=(3,3), padding="same"))
model.add(LeakyReLU(0.1))
model.add(MaxPool2D(pool_size=(2, 2), strides=None, padding='valid', data_format=None))
model.add(Dropout(0.25))
model.add(Conv2D(filters=128, kernel_size=(3,3), padding="same"))
model.add(LeakyReLU(0.1))
model.add(Conv2D(filters=128, kernel_size=(3,3), padding="same"))
model.add(LeakyReLU(0.1))
model.add(MaxPool2D(pool_size=(2, 2), strides=None, padding='valid', data_format=None))
model.add(Dropout(0.25))
model.add(Flatten())
 model.add(Dense(256))
model.add(LeakyReLU(0.1))
model.add(Dropout(0.5))
model.add(Dense(10, activation="softmax"))

<train_model>
train = train.drop(['id', 'text', 'author'], axis=1)
 test = test.drop(['id', 'text'], axis=1 )
epochs = 30
batch_size = 64
start = time.time()
history = model.fit_generator(datagen.flow(X_train,Y_train, batch_size=batch_size),
validation_data =(X_dev, Y_dev),
epochs = epochs,
steps_per_epoch=X_train.shape[0] // batch_size 
)
 end = time.time()
print("Total time training was:")
 print(end - start)
<save_to_csv>
predictions = model.predict(test)
 results = np.argmax(predictions, axis = 1)
 print(results)
 results = pd.Series(results, name="Label")
 submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),results],axis = 1)
 submission.to_csv("output.csv",index=False )<import_modules>
NUM_FEATURES = 100
model = Word2Vec(train_text + test_text, min_count=2, size=NUM_FEATURES, window=4, sg=1, alpha=1e-4, workers=4 )
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from keras.utils import to_categorical
from keras.preprocessing.image import ImageDataGenerator<load_from_csv>
len(model.wv.vocab )
np.random.seed(1)
 X_raw = pd.read_csv(".. /input/digit-recognizer/train.csv")
 X_test_raw = pd.read_csv(".. /input/digit-recognizer/test.csv")
 y = X_raw["label"]
X = X_raw.drop(labels = ["label"],axis = 1)
 X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.20, random_state=1)
 WIDTH=28
HEIGHT=28
NUM_CLASSES=10
X_train = X_train.values
y_train = y_train.values
X_valid = X_valid.values
y_valid = y_valid.values
y_train_oh = to_categorical(y_train, num_classes = NUM_CLASSES)
 y_valid_oh = to_categorical(y_valid, num_classes = NUM_CLASSES)
 X_test = X_test_raw.values<concatenate>
def get_feature_vec(tokens, num_features, model):
featureVec = np.zeros(shape=(1, num_features), dtype='float32')
 missed = 0
for word in tokens:
try:
featureVec = np.add(featureVec, model[word])
 except KeyError:
missed += 1
pass
if len(tokens)- missed == 0:
return np.zeros(shape=(num_features), dtype='float32')
 return np.divide(featureVec, len(tokens)- missed ).squeeze()
def reshape_channel(X):
return X.reshape(-1,HEIGHT,WIDTH,1)
 def preprocessData(X):
X = reshape_channel(X)
 return X<choose_model_class>
train_vectors = []
for i in train_text:
train_vectors.append(get_feature_vec([lemmatizer.lemmatize(word.lower())for word in alpha_tokenizer.tokenize(i)if word.lower() not in stop], NUM_FEATURES, model))
optimizer = Adam(learning_rate=0.0001, beta_1=0.9, beta_2=0.9999, amsgrad=True)
 model = Sequential()
model.add(Conv2D(filters = 64, kernel_size =(5,5),padding = 'same',activation ='relu',input_shape =(HEIGHT,WIDTH,1)))
 model.add(Conv2D(filters = 64, kernel_size =(5,5),padding = 'same',activation ='relu'))
model.add(Conv2D(filters = 64, kernel_size =(5,5),padding = 'same',activation ='relu'))
model.add(MaxPool2D(pool_size=(5,5),padding='same'))
model.add(Dropout(0.2))
model.add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'same',activation ='relu'))
model.add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'same',activation ='relu'))
model.add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'same',activation ='relu'))
model.add(MaxPool2D(pool_size=(3,3),padding='same'))
model.add(Dropout(0.2))
model.add(Flatten())
 model.add(Dense(256, activation = "relu"))
model.add(Dense(256, activation = "relu"))
model.add(Dropout(0.2))
model.add(Dense(NUM_CLASSES, activation = "softmax"))
model.compile(optimizer ='adam', loss = "categorical_crossentropy", metrics=["accuracy"])
 model.summary()<train_model>
test_vectors = []
for i in test_text:
test_vectors.append(get_feature_vec([lemmatizer.lemmatize(word.lower())for word in alpha_tokenizer.tokenize(i)if word.lower() not in stop], NUM_FEATURES, model))
datagen_with_aug = ImageDataGenerator(
 rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1,
rescale=1./255, 
)
 datagen_with_aug.fit(preprocessData(X_train))
datagen_no_aug = ImageDataGenerator()
train_generator = datagen_with_aug.flow(preprocessData(X_train), y_train_oh, batch_size=128)
 valid_generator = datagen_with_aug.flow(preprocessData(X_valid), y_valid_oh,batch_size=128 )<train_model>
full_vectors = []
for i in train_text + test_text:
full_vectors.append(get_feature_vec([lemmatizer.lemmatize(word.lower())for word in alpha_tokenizer.tokenize(i)if word.lower() not in stop], NUM_FEATURES, model))
keras.initializers.RandomNormal(mean=0.0, stddev=0.1, seed=1)
 hist = model.fit_generator(train_generator,validation_data=valid_generator,callbacks=[lrate],epochs=n_epoch,steps_per_epoch=(X_train.shape[0]/128)*2)
<save_to_csv>
svd = TruncatedSVD(n_components=30, algorithm='arpack')
 svd.fit(full_vectors)
 train_svd = pd.DataFrame(svd.transform(np.array(train_vectors)))
 test_svd = pd.DataFrame(svd.transform(np.array(test_vectors)))
 train_svd.columns = ['W2V_' + str(i)for i in range(30)]
test_svd.columns = ['W2V_' + str(i)for i in range(30)]
train = pd.concat([train, train_svd], axis=1)
 test = pd.concat([test, test_svd], axis=1 )
y_pred = model.predict(preprocessData(X_test))
y_pred = np.argmax(y_pred,axis = 1)
 submission = pd.DataFrame({'ImageId':range(1,28001),'Label':y_pred})
 submission.to_csv('submission.csv',index=False )<import_modules>
pred_full_test = 0
pred_train = np.zeros([train.shape[0], 3])
 for dev_index, val_index in KFold(n_splits=5, shuffle=True, random_state=42 ).split(train):
dev_X, val_X = train.loc[dev_index], train.loc[val_index]
dev_y, val_y = y_train[dev_index], y_train[val_index]
xgtrain = xgb.DMatrix(dev_X, dev_y)
 xgtest = xgb.DMatrix(test)
 model = xgb.train(params=list(params.items()), dtrain=xgtrain, num_boost_round=1000)
 predictions = model.predict(xgtest, ntree_limit=model.best_ntree_limit)
 pred_full_test = pred_full_test + predictions
pred_full_test = pred_full_test / 5 .
from fastai import *
from fastai.text import *
import pandas as pd<load_from_csv>
author = pd.DataFrame(pred_full_test)
 final = pd.DataFrame()
final['id'] = test_id
final['EAP'] = author[0]
final['HPL'] = author[1]
final['MWS'] = author[2]
final.to_csv('submission.csv', sep=',',index=False )