Kaggle competition: Digit Recognizer
def remove_URL(text):
url = re.compile(r'https?://\S+|www\.\S+')
 return url.sub(r'',text )<feature_engineering>
%matplotlib inline
np.random.seed(42)
df_train['text']=df_train['text'].apply(lambda x : remove_URL(x))<choose_model_class>
train_data = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')
 test_data = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')
 submission=pd.read_csv('/kaggle/input/digit-recognizer/sample_submission.csv' )
aug_w2v = naw.WordEmbsAug(
 model_type='glove', model_path='.. /input/glove-global-vectors-for-word-representation/glove.6B.100d.txt',
action="substitute")
<normalization>
train_data=".. /input/digit-recognizer/train.csv"
test_data = ".. /input/digit-recognizer/test.csv"
aug_w2v.aug_p=0.2
print("Augmented Text:")
 for ii in range(5):
augmented_text = aug_w2v.augment(text)
 print(augmented_text )<split>
Xtrain,Xtest,ytrain,ytest=train_test_split(raw_data[:,1:], raw_data[:,0],test_size=0.2,
stratify=raw_data[:,0],
random_state=42)
train,valid=train_test_split(df_train,test_size=0.15)
 print('Shape of train',train.shape)
 print("Shape of Validation ",valid.shape )<categorify>
Xtrain=Xtrain/255.
 Xtest=Xtest/255 .
def augment_text(df,samples=300,pr=0.2):
aug_w2v.aug_p=pr
new_text=[]
df_n=df[df.target==1].reset_index(drop=True)
 for i in tqdm(np.random.randint(0,len(df_n),samples)) :
text = df_n.iloc[i]['text']
augmented_text = aug_w2v.augment(text)
 new_text.append(augmented_text)
 new=pd.DataFrame({'text':new_text,'target':1})
 df=shuffle(df.append(new ).reset_index(drop=True))
return df 
<categorify>
ytrain= to_categorical(ytrain, num_classes = 10)
 ytest= to_categorical(ytest, num_classes = 10 )
train = augment_text(train,samples=400)
 tweet = train.append(valid ).reset_index(drop=True )<concatenate>
Xtrain=Xtrain.astype("float32" ).reshape(-1,28,28,1)
 Xtest=Xtest.astype("float32" ).reshape(-1,28,28,1 )
df=pd.concat([tweet,df_test] )<remove_duplicates>
train_gen=ImageDataGenerator(rotation_range=20,
width_shift_range=0.1,
height_shift_range=0.1,
zoom_range=0.1,
shear_range=0.2
)
def create_corpus(df):
corpus=[]
for tweet in tqdm(df['text']):
words=[word.lower() for word in word_tokenize(tweet)if(( word.isalpha() ==1)&(word not in stop)) ]
corpus.append(words)
 return corpus 
<statistical_test>
def model() :
model=tf.keras.Sequential()
model.add(tf.keras.layers.Conv2D(16,(3,3), activation="relu",padding="SAME",input_shape=(28,28,1)))
 model.add(BatchNormalization())
 model.add(tf.keras.layers.Conv2D(32,(3,3),activation="relu",padding="SAME"))
model.add(BatchNormalization())
 model.add(tf.keras.layers.MaxPool2D(strides=(2,2)))
 model.add(Dropout(0.25))
model.add(tf.keras.layers.Conv2D(32,(3,3),padding='SAME',activation='relu'))
model.add(BatchNormalization())
 model.add(tf.keras.layers.Conv2D(64,(3,3),padding='SAME',activation='relu'))
model.add(BatchNormalization())
 model.add(tf.keras.layers.MaxPool2D(strides=(2,2)))
 model.add(Dropout(0.25))
model.add(tf.keras.layers.Flatten())
 model.add(tf.keras.layers.Dense(512,activation="relu"))
model.add(Dropout(0.25))
model.add(tf.keras.layers.Dense(256,activation="relu"))
model.add(Dropout(0.5))
model.add(tf.keras.layers.Dense(10,activation="softmax"))
return model
corpus=create_corpus(df )<load_from_csv>
annealer = LearningRateScheduler(lambda x: 1e-3 * 0.9 ** x )
embedding_dict={}
with open('.. /input/glove-global-vectors-for-word-representation/glove.6B.100d.txt','r')as f:
for line in f:
values=line.split()
word=values[0]
vectors=np.asarray(values[1:],'float32')
 embedding_dict[word]=vectors
f.close()<categorify>
model.compile(optimizer=Adam(lr=1e-4),loss='categorical_crossentropy',
metrics=['accuracy'] )
MAX_LEN=50
tokenizer_obj=Tokenizer()
tokenizer_obj.fit_on_texts(corpus)
 sequences=tokenizer_obj.texts_to_sequences(corpus)
 tweet_pad=pad_sequences(sequences,maxlen=MAX_LEN,truncating='post',padding='post' )<count_unique_values>
class MyCallback(tf.keras.callbacks.Callback):
def on_epoch_end(self,epochs,logs={}):
if(logs.get('accuracy')>0.99):
print()
print("Stopping Training")
 self.model.stop_training=True
word_index=tokenizer_obj.word_index
print('Number of unique words:',len(word_index))<define_variables>
history=model.fit_generator(train_gen.flow(Xtrain,ytrain,batch_size=16),
validation_data=(Xtest[:1000,:], ytest[:1000,:]),
epochs=30,callbacks=[annealer], steps_per_epoch=500
)
num_words=len(word_index)+1
embedding_matrix=np.zeros(( num_words,100))
for word,i in tqdm(word_index.items()):
if i > num_words:
continue
emb_vec=embedding_dict.get(word)
 if emb_vec is not None:
embedding_matrix[i]=emb_vec 
<choose_model_class>
model.evaluate(Xtest,ytest )
model = Sequential()
embedding=Embedding(num_words,100,embeddings_initializer=Constant(embedding_matrix),
input_length=MAX_LEN,trainable=False)
 model.add(embedding)
 model.add(SimpleRNN(100))
model.add(Dense(1, activation='sigmoid'))
optimzer=Adam(learning_rate=1e-5)
 model.compile(loss='binary_crossentropy',optimizer=optimzer,metrics=['accuracy'])
 model.summary() 
<split>
testing = np.loadtxt(test_data, skiprows=1, dtype='int', delimiter=',')
 test = testing.astype("float32")
 test = testing.reshape(-1, 28, 28, 1)/255 .
train_df=tweet_pad[:tweet.shape[0]]
test_df=tweet_pad[tweet.shape[0]:]<prepare_x_and_y>
ypred=np.argmax(model.predict(test),axis=-1 )
<train_model><EOS> 
submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),results],axis = 1)
 submission.to_csv("MNIST_digit_recog.csv",index=False )