Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<count_values>
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from keras.utils import to_categorical
from keras.preprocessing.image import ImageDataGenerator
len(train["location"].value_counts() )<remove_duplicates>
np.random.seed(1)
 X_raw = pd.read_csv(".. /input/digit-recognizer/train.csv")
 X_test_raw = pd.read_csv(".. /input/digit-recognizer/test.csv")
 y = X_raw["label"]
X = X_raw.drop(labels = ["label"],axis = 1)
 X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.25, random_state=0)
 WIDTH=28
HEIGHT=28
NUM_CLASSES=10
X_train = X_train.values
y_train = y_train.values
X_valid = X_valid.values
y_valid = y_valid.values
y_train_oh = to_categorical(y_train, num_classes = NUM_CLASSES)
 y_valid_oh = to_categorical(y_valid, num_classes = NUM_CLASSES)
 X_test = X_test_raw.values
def create_corpus(target):
corpus = []
for w in train.loc[train["target"] == target]["text"].str.split() :
for i in w:
corpus.append(i)
 return corpus
def create_corpus_dict(target):
corpus = create_corpus(target)
 stop_dict = defaultdict(int)
 for word in corpus:
if word in stopwords:
stop_dict[word] += 1
return sorted(stop_dict.items() , key=lambda x:x[1], reverse=True )<count_duplicates>
def scaleData(X):
n_max = X_train.max()
X = X/n_max
return X
def reshape_channel(X):
return np.expand_dims(X.reshape(-1,HEIGHT,WIDTH),-1)
 def preprocessData(X):
X = scaleData(X)
 X = reshape_channel(X)
 return X
corpus_disaster, corpus_non_disaster = create_corpus(1), create_corpus(0)
 counter_disaster, counter_non_disaster = Counter(corpus_disaster), Counter(corpus_non_disaster)
 x_disaster, y_disaster, x_non_disaster, y_non_disaster = [], [], [], []
counter = 0
for word, count in counter_disaster.most_common() [0:100]:
if(word not in stopwords and counter < 15):
counter += 1
x_disaster.append(word)
 y_disaster.append(count)
 counter = 0
for word, count in counter_non_disaster.most_common() [0:100]:
if(word not in stopwords and counter < 15):
counter += 1
x_non_disaster.append(word)
 y_non_disaster.append(count)
 fig, ax = plt.subplots(1,2, figsize=(20,8))
sns.barplot(x=y_disaster, y=x_disaster, orient='h', palette="Reds_d", ax=ax[0])
 sns.barplot(x=y_non_disaster, y=x_non_disaster, orient='h', palette="Blues_d", ax=ax[1])
 ax[0].set_title("Top 15 Non-Stopwords - Disaster Tweets")
 ax[0].set_xlabel("Word Frequency")
 ax[1].set_title("Top 15 Non-Stopwords - Non-Disaster Tweets")
 ax[1].set_xlabel("Word Frequency")
 plt.tight_layout()
plt.show()<feature_engineering>
optimizer = Adam(learning_rate=0.0001, beta_1=0.9, beta_2=0.9999, amsgrad=True)
 model = Sequential()
model.add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'same',activation ='relu',use_bias=True,input_shape =(HEIGHT,WIDTH,1)))
 model.add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'same',activation ='relu',use_bias=True))
model.add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'same',activation ='relu',use_bias=True))
model.add(MaxPool2D(pool_size=(3,3), strides=(3,3)))
 model.add(Dropout(0.2))
model.add(Conv2D(filters = 64, kernel_size =(5,5),padding = 'same',activation ='relu',use_bias=True))
model.add(Conv2D(filters = 64, kernel_size =(5,5),padding = 'same',activation ='relu',use_bias=True))
model.add(Conv2D(filters = 64, kernel_size =(5,5),padding = 'same',activation ='relu',use_bias=True))
model.add(MaxPool2D(pool_size=(5,5), strides=(5,5)))
 model.add(Flatten())
 model.add(Dropout(0.2))
model.add(Dense(64, activation = "relu"))
model.add(Dense(64, activation = "relu"))
model.add(Dense(NUM_CLASSES, activation = "softmax"))
model.compile(optimizer ='adam', loss = "categorical_crossentropy", metrics=["accuracy"])
 model.summary()
def bigrams(target):
corpus = train[train["target"] == target]["text"]
count_vec = CountVectorizer(ngram_range=(2, 2)).fit(corpus)
 bag_of_words = count_vec.transform(corpus)
 sum_words = bag_of_words.sum(axis=0)
 words_freq = [(word, sum_words[0, idx])for word, idx in count_vec.vocabulary_.items() ]
words_freq =sorted(words_freq, key = lambda x: x[1], reverse=True)
 return words_freq<feature_engineering>
datagen = ImageDataGenerator(
 rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1, 
)
 it_train = datagen.flow(preprocessData(X_train), y_train_oh)
 it_valid = datagen.flow(preprocessData(X_valid), y_valid_oh )
def remove_pattern(input_txt, pattern):
r = re.findall(pattern, input_txt)
 for i in r:
input_txt = re.sub(i, '', input_txt)
 return input_txt<feature_engineering>
hist = model.fit_generator(it_train,validation_data=it_valid,callbacks=[lrate],epochs=n_epoch)
<feature_engineering><EOS> 
y_pred = model.predict(preprocessData(X_test))
y_pred = np.argmax(y_pred,axis = 1)
 showImg(X_test,y_pred,4,4)
 submission = pd.DataFrame({'ImageId':range(1,28001),'Label':y_pred})
 submission.to_csv('submission.csv',index=False )