Kaggle competition: Spooky Author Identification
<SOS> metric: MulticlassLoss Kaggle data source: spooky-author-identification<save_to_csv>
np.random.seed(7 )
test_set =(test.values ).reshape(-1, 28, 28 , 1 ).astype('float32')
 res = model.predict(test_set)
 res = numpy.argmax(res,axis = 1)
 res = pd.Series(res, name="Label")
 submission = pd.concat([pd.Series(range(1 ,28001),name = "ImageId"), res],axis = 1)
 submission.to_csv("cnn_mnist_datagen.csv",index=False)
 submission.head(10 )<load_from_csv>
df = pd.read_csv('./.. /input/train.csv')
 a2c = {'EAP': 0, 'HPL' : 1, 'MWS' : 2}
y = np.array([a2c[a] for a in df.author])
 y = to_categorical(y )
print(os.listdir(".. /input"))
size_img = 28
threshold_color = 100 / 255
file = open(".. /input/digit-recognizer/train.csv")
 data_train = pd.read_csv(file)
 y_train = np.array(data_train.iloc[:, 0])
 x_train = np.array(data_train.iloc[:, 1:])
 file = open(".. /input/digit-recognizer/test.csv")
 data_test = pd.read_csv(file)
 x_test = np.array(data_test)
 n_features_train = x_train.shape[1]
n_samples_train = x_train.shape[0]
n_features_test = x_test.shape[1]
n_samples_test = x_test.shape[0]
print(n_features_train, n_samples_train, n_features_test, n_samples_test)
 print(x_train.shape, y_train.shape, x_test.shape )<feature_engineering>
def preprocess(text):
text = text.replace("' ", " ' ")
 signs = set(',.:;"?!')
 prods = set(text)& signs
if not prods:
return text
for sign in prods:
text = text.replace(sign, ' {} '.format(sign))
return text
def int2float_grey(x):
x = x / 255
return x<merge>
def create_docs(df, n_gram_max=2):
def add_ngram(q, n_gram_max):
ngrams = []
for n in range(2, n_gram_max+1):
for w_index in range(len(q)-n+1):
ngrams.append('--'.join(q[w_index:w_index+n]))
return q + ngrams
docs = []
for doc in df.text:
doc = preprocess(doc ).split()
docs.append(' '.join(add_ngram(doc, n_gram_max)))
 return docs
def find_left_edge(x):
edge_left = []
n_samples = x.shape[0]
for k in range(n_samples):
for j in range(size_img):
for i in range(size_img):
if(x[k, size_img*i+j] >= threshold_color):
edge_left.append(j)
 break
if(len(edge_left)> k):
break
return edge_left<categorify>
min_count = 2
docs = create_docs(df)
 tokenizer = Tokenizer(lower=False, filters='')
 tokenizer.fit_on_texts(docs)
 num_words = sum([1 for _, v in tokenizer.word_counts.items() if v >= min_count])
 tokenizer = Tokenizer(num_words=num_words, lower=False, filters='')
 tokenizer.fit_on_texts(docs)
 docs = tokenizer.texts_to_sequences(docs)
 maxlen = 256
docs = pad_sequences(sequences=docs, maxlen=maxlen )
def find_top_edge(x):
edge_top = []
n_samples = x.shape[0]
for k in range(n_samples):
for i in range(size_img):
for j in range(size_img):
if(x[k, size_img*i+j] >= threshold_color):
edge_top.append(i)
 break
if(len(edge_top)> k):
break
return edge_top<normalization>
input_dim = np.max(docs)+ 1
embedding_dims = 20
def stretch_image(x):
edge_left = find_left_edge(x)
 edge_right = find_right_edge(x)
 edge_top = find_top_edge(x)
 edge_bottom = find_bottom_edge(x)
 n_samples = x.shape[0]
x = x.reshape(n_samples, size_img, size_img)
 for i in range(n_samples):
x[i] = transform.resize(x[i][edge_top[i]:edge_bottom[i]+1, edge_left[i]:edge_right[i]+1],(size_img, size_img))
x = x.reshape(n_samples, size_img ** 2)
 show_img(x )<train_model>
def create_model(embedding_dims=20, optimizer='adam'):
model = Sequential()
model.add(Embedding(input_dim=input_dim, output_dim=embedding_dims))
model.add(GlobalAveragePooling1D())
 model.add(Dense(3, activation='softmax'))
model.compile(loss='categorical_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
 return model
def get_threshold(x_train, x_test):
selector = VarianceThreshold(threshold=0 ).fit(x_train)
 x_train = selector.transform(x_train)
 x_test = selector.transform(x_test)
 print(x_train.shape)
 return x_train, x_test<normalization>
epochs = 25
x_train, x_test, y_train, y_test = train_test_split(docs, y, test_size=0.2)
 model = create_model()
hist = model.fit(x_train, y_train,
batch_size=16,
validation_data=(x_test, y_test),
epochs=epochs,
callbacks=[EarlyStopping(patience=2, monitor='val_loss')] )
def get_pca(x_train, x_test):
pca = PCA(n_components=0.95)
 pca.fit(x_train)
 x_train = pca.transform(x_train)
 x_test = pca.transform(x_test)
 print(x_train.shape, x_test.shape)
 return x_train, x_test<compute_test_metric>
docs = create_docs(df)
 tokenizer = Tokenizer(lower=True, filters='')
 tokenizer.fit_on_texts(docs)
 num_words = sum([1 for _, v in tokenizer.word_counts.items() if v >= min_count])
 tokenizer = Tokenizer(num_words=num_words, lower=True, filters='')
 tokenizer.fit_on_texts(docs)
 docs = tokenizer.texts_to_sequences(docs)
 maxlen = 256
docs = pad_sequences(sequences=docs, maxlen=maxlen)
 input_dim = np.max(docs)+ 1
x_train = int2float_grey(x_train)
 x_test = int2float_grey(x_test)
<predict_on_test>
epochs = 16
x_train, x_test, y_train, y_test = train_test_split(docs, y, test_size=0.2)
 model = create_model()
hist = model.fit(x_train, y_train,
batch_size=16,
validation_data=(x_test, y_test),
epochs=epochs,
callbacks=[EarlyStopping(patience=2, monitor='val_loss')] )
def general_function(mod_name, model_name):
y_pred = model_train_predict(mod_name, model_name)
 output_prediction(y_pred, model_name )<find_best_model_class>
test_df = pd.read_csv('.. /input/test.csv')
 docs = create_docs(test_df)
 docs = tokenizer.texts_to_sequences(docs)
 docs = pad_sequences(sequences=docs, maxlen=maxlen)
 y = model.predict_proba(docs)
 result = pd.read_csv('.. /input/sample_submission.csv')
 for a, i in a2c.items() :
result[a] = y[:, i]
<compute_test_metric><EOS> 
result.to_csv('fastText_result.csv', index=False )