Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<import_modules>
%matplotlib inline
np.random.seed(2 )
import pandas as pd
import numpy as np
import nltk
from nltk.corpus import stopwords
from nltk.stem import SnowballStemmer
import re
from string import punctuation<import_modules>
test = pd.read_csv('.. /input/digit-recognizer/test.csv')
import tensorflow as tf
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences
from tensorflow.keras.models import Model, Sequential
from tensorflow.keras.layers import Dense, Input, LSTM, Embedding, Bidirectional
from tensorflow.keras import layers, utils, callbacks, optimizers, regularizers
from tqdm import tqdm<define_variables>
( X_train, Y_train),(X_val, Y_val)= mnist.load_data()
print("Num GPUs Available: ", len(tf.config.list_physical_devices('GPU')) )<load_from_csv>
X_train = X_train.reshape(-1, 28, 28, 1)
 X_val = X_val.reshape(-1, 28, 28, 1)
 X_train = X_train.astype('float32')
 X_val = X_val.astype('float32')
 test = test.values.reshape(-1, 28, 28, 1)
 print(X_train.shape,', ',X_val.shape,', ', test.shape )
train = pd.read_csv(".. /input/quora-question-pairs/train.csv.zip")
 test = pd.read_csv(".. /input/quora-question-pairs/test.csv" )<drop_column>
Y_train = to_categorical(Y_train, num_classes = 10)
 Y_val = to_categorical(Y_val, num_classes = 10 )
def clean_dataframe_train(train):
stop_words = ['the','a','an','and','but','if','or','because','as','what','which','this','that','these','those','then',
'just','so','than','such','both','through','about','for','is','of','while','during','to','What','Which',
'Is','If','While','This']
def text_to_wordlist(text, remove_stop_words=True, stem_words=False):
text = re.sub(r"[^A-Za-z0-9]", " ", text)
 text = re.sub(r"what's", "", text)
 text = re.sub(r"What's", "", text)
 text = re.sub(r"'s", " ", text)
 text = re.sub(r"'ve", " have ", text)
 text = re.sub(r"can't", "cannot ", text)
 text = re.sub(r"n't", " not ", text)
 text = re.sub(r"I'm", "I am", text)
 text = re.sub(r" m ", " am ", text)
 text = re.sub(r"'re", " are ", text)
 text = re.sub(r"'d", " would ", text)
 text = re.sub(r"'ll", " will ", text)
 text = re.sub(r"60k", " 60000 ", text)
 text = re.sub(r" e g ", " eg ", text)
 text = re.sub(r" b g ", " bg ", text)
 text = re.sub(r"\0s", "0", text)
 text = re.sub(r" 9 11 ", "911", text)
 text = re.sub(r"e-mail", "email", text)
 text = re.sub(r"\s{2,}", " ", text)
 text = re.sub(r"quikly", "quickly", text)
 text = re.sub(r" usa ", " America ", text)
 text = re.sub(r" USA ", " America ", text)
 text = re.sub(r" u s ", " America ", text)
 text = re.sub(r" uk ", " England ", text)
 text = re.sub(r" UK ", " England ", text)
 text = re.sub(r"india", "India", text)
 text = re.sub(r"switzerland", "Switzerland", text)
 text = re.sub(r"china", "China", text)
 text = re.sub(r"chinese", "Chinese", text)
 text = re.sub(r"imrovement", "improvement", text)
 text = re.sub(r"intially", "initially", text)
 text = re.sub(r"quora", "Quora", text)
 text = re.sub(r" dms ", "direct messages ", text)
 text = re.sub(r"demonitization", "demonetization", text)
 text = re.sub(r"actived", "active", text)
 text = re.sub(r"kms", " kilometers ", text)
 text = re.sub(r"KMs", " kilometers ", text)
 text = re.sub(r" cs ", " computer science ", text)
 text = re.sub(r" upvotes ", " up votes ", text)
 text = re.sub(r" iPhone ", " phone ", text)
 text = re.sub(r"\0rs ", " rs ", text)
 text = re.sub(r"calender", "calendar", text)
 text = re.sub(r"ios", "operating system", text)
 text = re.sub(r"gps", "GPS", text)
 text = re.sub(r"gst", "GST", text)
 text = re.sub(r"programing", "programming", text)
 text = re.sub(r"bestfriend", "best friend", text)
 text = re.sub(r"dna", "DNA", text)
 text = re.sub(r"III", "3", text)
 text = re.sub(r"the US", "America", text)
 text = re.sub(r"Astrology", "astrology", text)
 text = re.sub(r"Method", "method", text)
 text = re.sub(r"Find", "find", text)
 text = re.sub(r"banglore", "Banglore", text)
 text = re.sub(r" J K ", " JK ", text)
 text = ''.join([c for c in text if c not in punctuation])
 if remove_stop_words:
text = text.split()
text = [w for w in text if not w in stop_words]
text = " ".join(text)
 if stem_words:
text = text.split()
stemmer = SnowballStemmer('english')
 stemmed_words = [stemmer.stem(word)for word in text]
text = " ".join(stemmed_words)
 return(text)
 def process_questions(question_list, questions, question_list_name, dataframe):
for question in questions:
question_list.append(text_to_wordlist(str(question)))
 if len(question_list)% 100000 == 0:
progress = len(question_list)/len(dataframe)* 100
print("{} is {}% complete.".format(question_list_name, round(progress, 1)))
 train_question1 = []
process_questions(train_question1, train.question1, 'train_question1', train)
 train_question2 = []
process_questions(train_question2, train.question2, 'train_question2', train)
 train["question1"] = train_question1
train["question2"] = train_question2
return train<drop_column>
model = Sequential()
model.add(Conv2D(filters = 32, kernel_size =(5, 5), padding = "Same", activation = "relu",
kernel_initializer = 'he_uniform', input_shape =(28, 28, 1)))
 model.add(Conv2D(filters = 32, kernel_size =(5, 5), padding = "Same", activation = "relu",
kernel_initializer = 'he_uniform'))
model.add(MaxPooling2D(pool_size =(2, 2)))
 model.add(Dropout(0.25))
model.add(Conv2D(filters = 64, kernel_size =(3, 3), padding = "Same", activation = "relu",
kernel_initializer = 'he_uniform'))
model.add(Conv2D(filters = 64, kernel_size =(3, 3), padding = "Same", activation = "relu",
kernel_initializer = 'he_uniform'))
model.add(MaxPooling2D(pool_size =(2, 2), strides =(2, 2)))
 model.add(Dropout(0.25))
model.add(Flatten())
 model.add(Dense(256, activation="relu", kernel_initializer = 'he_uniform'))
model.add(Dropout(0.5))
model.add(Dense(units = 10, activation="softmax", kernel_initializer = 'he_uniform'))
def clean_dataframe_test(test):
stop_words = ['the','a','an','and','but','if','or','because','as','what','which','this','that','these','those','then',
'just','so','than','such','both','through','about','for','is','of','while','during','to','What','Which',
'Is','If','While','This']
def text_to_wordlist(text, remove_stop_words=True, stem_words=False):
text = re.sub(r"[^A-Za-z0-9]", " ", text)
 text = re.sub(r"what's", "", text)
 text = re.sub(r"What's", "", text)
 text = re.sub(r"'s", " ", text)
 text = re.sub(r"'ve", " have ", text)
 text = re.sub(r"can't", "cannot ", text)
 text = re.sub(r"n't", " not ", text)
 text = re.sub(r"I'm", "I am", text)
 text = re.sub(r" m ", " am ", text)
 text = re.sub(r"'re", " are ", text)
 text = re.sub(r"'d", " would ", text)
 text = re.sub(r"'ll", " will ", text)
 text = re.sub(r"60k", " 60000 ", text)
 text = re.sub(r" e g ", " eg ", text)
 text = re.sub(r" b g ", " bg ", text)
 text = re.sub(r"\0s", "0", text)
 text = re.sub(r" 9 11 ", "911", text)
 text = re.sub(r"e-mail", "email", text)
 text = re.sub(r"\s{2,}", " ", text)
 text = re.sub(r"quikly", "quickly", text)
 text = re.sub(r" usa ", " America ", text)
 text = re.sub(r" USA ", " America ", text)
 text = re.sub(r" u s ", " America ", text)
 text = re.sub(r" uk ", " England ", text)
 text = re.sub(r" UK ", " England ", text)
 text = re.sub(r"india", "India", text)
 text = re.sub(r"switzerland", "Switzerland", text)
 text = re.sub(r"china", "China", text)
 text = re.sub(r"chinese", "Chinese", text)
 text = re.sub(r"imrovement", "improvement", text)
 text = re.sub(r"intially", "initially", text)
 text = re.sub(r"quora", "Quora", text)
 text = re.sub(r" dms ", "direct messages ", text)
 text = re.sub(r"demonitization", "demonetization", text)
 text = re.sub(r"actived", "active", text)
 text = re.sub(r"kms", " kilometers ", text)
 text = re.sub(r"KMs", " kilometers ", text)
 text = re.sub(r" cs ", " computer science ", text)
 text = re.sub(r" upvotes ", " up votes ", text)
 text = re.sub(r" iPhone ", " phone ", text)
 text = re.sub(r"\0rs ", " rs ", text)
 text = re.sub(r"calender", "calendar", text)
 text = re.sub(r"ios", "operating system", text)
 text = re.sub(r"gps", "GPS", text)
 text = re.sub(r"gst", "GST", text)
 text = re.sub(r"programing", "programming", text)
 text = re.sub(r"bestfriend", "best friend", text)
 text = re.sub(r"dna", "DNA", text)
 text = re.sub(r"III", "3", text)
 text = re.sub(r"the US", "America", text)
 text = re.sub(r"Astrology", "astrology", text)
 text = re.sub(r"Method", "method", text)
 text = re.sub(r"Find", "find", text)
 text = re.sub(r"banglore", "Banglore", text)
 text = re.sub(r" J K ", " JK ", text)
 text = ''.join([c for c in text if c not in punctuation])
 if remove_stop_words:
text = text.split()
text = [w for w in text if not w in stop_words]
text = " ".join(text)
 if stem_words:
text = text.split()
stemmer = SnowballStemmer('english')
 stemmed_words = [stemmer.stem(word)for word in text]
text = " ".join(stemmed_words)
 return(text)
 def process_questions(question_list, questions, question_list_name, dataframe):
for question in questions:
question_list.append(text_to_wordlist(str(question)))
 if len(question_list)% 100000 == 0:
progress = len(question_list)/len(dataframe)* 100
print("{} is {}% complete.".format(question_list_name, round(progress, 1)))
 test_question1 = []
process_questions(test_question1, test.question1, 'test_question1', test)
 test_question2 = []
process_questions(test_question2, test.question2, 'test_question2', test)
 test["question1"] = test_question1
test["question2"] = test_question2
return test<create_dataframe>
optimizer = RMSprop(lr = 0.001, rho = 0.9, epsilon = 1e-08, decay = 0.0)
 model.compile(optimizer = optimizer , loss = 'categorical_crossentropy', metrics = ['accuracy'] )
train = clean_dataframe_train(train)
 test = clean_dataframe_test(test )<define_variables>
checkpoint = ModelCheckpoint('./mod_best.hdf5',monitor = 'val_loss', mode = "min", verbose = 1, save_best_model = True)
 earlystop = EarlyStopping(monitor = 'val_loss', patience = 3, min_delta = 0,verbose =1, restore_best_weights = True)
 reduce_lr = ReduceLROnPlateau(monitor='val_loss', patience=3, verbose=1, factor=0.5, min_lr=0.00001)
 callbacks = [reduce_lr,checkpoint]
MAX_SEQUENCE_LENGTH = 100
EMBEDDING_DIM = 300<data_type_conversions>
datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False)
 datagen.fit(X_train )
corpus = []
feats = ['question1','question2']
for f in feats:
train[f] = train[f].astype(str)
 test[f] = test[f].astype(str)
 corpus+=train[f].values.tolist()
tokenizer = Tokenizer()
tokenizer.fit_on_texts(corpus)
 X_q1 = tokenizer.texts_to_sequences(train['question1'])
 X_q2 = tokenizer.texts_to_sequences(train['question2'])
 X_test_q1 = tokenizer.texts_to_sequences(test['question1'])
 X_test_q2 = tokenizer.texts_to_sequences(test['question2'])
 X_q1 = pad_sequences(X_q1, maxlen=MAX_SEQUENCE_LENGTH)
 X_q2 = pad_sequences(X_q2, maxlen=MAX_SEQUENCE_LENGTH)
 X_test_q1 = pad_sequences(X_test_q1, maxlen=MAX_SEQUENCE_LENGTH)
 X_test_q2 = pad_sequences(X_test_q2, maxlen=MAX_SEQUENCE_LENGTH)
 y = train['is_duplicate'].values
word_index = tokenizer.word_index
nb_words = len(word_index)+1<feature_engineering>
model = load_model("./mod_best.hdf5" )
def get_coefs(word, *arr):
return word, np.asarray(arr, dtype='float32')
 def load_embeddings(path):
with open(path)as f:
return dict(get_coefs(*line.strip().split(' ')) for line in tqdm(f))
def build_matrix(word_index, path):
embedding_index = load_embeddings(path)
 embedding_matrix = np.zeros(( len(word_index)+ 1, 300))
unknown_words = []
for word, i in word_index.items() :
try:
embedding_matrix[i] = embedding_index[word]
except KeyError:
unknown_words.append(word)
 return embedding_matrix, unknown_words
glove_path = '.. /input/glove840b300dtxt/glove.840B.300d.txt'
embedding_matrix,unknown_words = build_matrix(word_index,glove_path )<split>
batch_size = 128
epochs = 30
history = model.fit_generator(datagen.flow(X_train,Y_train, batch_size = batch_size),
epochs = epochs, validation_data = datagen.flow(X_val,Y_val, batch_size = batch_size),
steps_per_epoch = X_train.shape[0] // batch_size,
callbacks = callbacks )
X_train_q1,X_val_q1,X_train_q2,X_val_q2,y_train,y_val = train_test_split(X_q1,X_q2,y,train_size=0.8,random_state=1024)
 print(X_train_q1.shape,X_val_q1.shape)
 X_train = [X_train_q1,X_train_q2]
X_val = [X_val_q1,X_val_q2]
X_test = [X_test_q1,X_test_q2]<categorify>
score = model.evaluate(X_val, Y_val, verbose=0)
 print("Val_Loss :",score[0])
 print("Val_Acc :",score[1] )
input_q1 = Input(shape = X_train[0].shape[1])
 input_q2 = Input(shape = X_train[0].shape[1])
 embedding_layer = Embedding(nb_words,
EMBEDDING_DIM,
input_length = X_train[0].shape[1],
weights = [embedding_matrix],
trainable=False)
 embedded_sequences_q1 = embedding_layer(input_q1)
 embedded_sequences_q2 = embedding_layer(input_q2)
 bilstm_layer = Bidirectional(LSTM(64, return_sequences=False))
x1 = bilstm_layer(embedded_sequences_q1)
 x2 = bilstm_layer(embedded_sequences_q2)
 distance = layers.Concatenate()([x1, x2])
 x = Dense(64, activation="relu" )(distance)
 preds = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=[input_q1, input_q2], outputs=preds)
 model.compile(loss="binary_crossentropy",
optimizer="adam",
metrics=["accuracy"])
 model.summary()
utils.plot_model(model, show_shapes=True, expand_nested=True )<train_model>
probabilities = model.predict(test)
 y_pred = np.argmax(probabilities, axis=1)
 print(y_pred.shape )
<save_to_csv><EOS> 
submission = pd.DataFrame({'ImageId': np.arange(1, 28001), 'Label': y_pred})
 submission.to_csv("submission.csv", index = False)
 print("Your submission was successfully saved!" )