Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<compute_test_metric>
%matplotlib inline
warnings.filterwarnings('ignore')
 sns.set(style='white', context='notebook', palette='deep')
 np.random.seed(2)
 DEBUG = False
model.evaluate(x_test,y_test )<predict_on_test>
print("The Python version is %s.%s.%s." % sys.version_info[:3] )
y_pred = model.predict(x_test)
 X_new = x_test.reshape(x_test.shape[0], 28, 28)
 fig, axis = plt.subplots(4, 4, figsize=(12, 14))
for i, ax in enumerate(axis.flat):
ax.imshow(X_new[i], cmap='binary')
 ax.set(title = f"Real Number is {y_test[i].argmax() }
Predict Number is {y_pred[i].argmax() }");<load_from_csv>
%pwd
raw_test = test_data
sample = pd.read_csv('.. /input/Kannada-MNIST/sample_submission.csv' )<drop_column>
%time digit_train = pd.read_csv(".. /input/train.csv", header=0, sep=",")
 %time digit_test = pd.read_csv(".. /input/test.csv", header=0, sep="," )
sample_sub=sample
raw_test_id=raw_test.id
raw_test=raw_test.drop("id",axis="columns")
 raw_test=raw_test / 255
test=raw_test.values.reshape(-1,28,28,1)
 test.shape<save_to_csv>
features_train = digit_train.iloc[:,1:]
target_train = digit_train.iloc[:,0]
features_test = digit_test.iloc[:,0:]
sub=model.predict(test)
 sub=np.argmax(sub,axis=1)
 sample_sub['label']=sub
sample_sub.to_csv('submission.csv',index=False )<import_modules>
features_train = features_train/255.0
features_test = features_test/255.0
import numpy as np
import pandas as pd
from matplotlib import pyplot as plt<load_from_csv>
start_time = time.perf_counter()
train_results = []
test_results = []
k_range = list(range(1,5))
k_scores = []
for k in k_range:
knn = KNeighborsClassifier(n_neighbors=k, n_jobs=-1, algorithm='ball_tree', leaf_size=40, weights='uniform')
 scores = cross_val_score(knn, features_train, target_train, cv=10, scoring='accuracy', n_jobs=-1)
 k_scores.append(scores.mean())
 if DEBUG:
print(k_scores)
 print(time.perf_counter() - start_time, "seconds" )
origin_train = pd.read_csv('.. /input/Kannada-MNIST/train.csv')
 validate = pd.read_csv('.. /input/Kannada-MNIST/Dig-MNIST.csv' )<concatenate>
if DEBUG:
scores = pd.DataFrame(k_scores)
 print(scores )
train = pd.concat([origin_train, validate] )<train_model>
print("The optimal number of neighbors is %d." % optimal_k )
X_train = train.iloc[:, 1:]
y_train = train.iloc[:, 0]
print(X_train.shape, y_train.shape )<split>
start_time = time.perf_counter()
clf_knn = KNeighborsClassifier(n_neighbors=3, n_jobs=-1, algorithm='ball_tree', leaf_size=40, weights='uniform')
 print(clf_knn)
 clf_knn = clf_knn.fit(features_train, target_train)
 target_predicted_knn = clf_knn.predict(features_test)
 print(time.perf_counter() - start_time, "seconds" )
X_train, X_test, y_train, y_test = train_test_split(X_train.values, y_train.values, test_size=0.2, random_state=0 )<categorify>
start_time = time.perf_counter()
scores_knn = cross_val_score(clf_knn, features_train, target_train, cv=10, scoring='accuracy', n_jobs=-1)
 print("Cross Validation Score for each K",scores_knn)
 print("Accuracy: %0.2f(+/- %0.2f)" %(scores_knn.mean() , scores_knn.std() * 2))
print(time.perf_counter() - start_time, "seconds" )
def func_prep(data_prep):
data_prep_data=[]
for i in range(data_prep.shape[0]):
num=np.array(data_prep[i:i+1])
 num=num.ravel()
num=num.reshape(( 28,28))
data_prep_data.append(num)
 return np.array(data_prep_data)
 X_train=func_prep(X_train)
 X_test=np.array(X_test )<train_model>
digit_test['Label'] = pd.Series(target_predicted_knn)
 digit_test['ImageId'] = pd.Series(range(1,28001))
img_rows, img_cols = 28, 28
X_train = X_train.reshape(X_train.shape[0], img_rows, img_cols, 1)
 X_test = X_test.reshape(X_test.shape[0], img_rows, img_cols, 1)
 input_shape =(img_rows, img_cols, 1 )<data_type_conversions>
digit_test.to_csv('submission_knn.csv', columns=["ImageId","Label"], index=False )
X_train = X_train.astype('float32')
 X_test = X_test.astype('float32')
 X_train /= 255
X_test /= 255
print('X_train shape:', X_train.shape)
 print(X_train.shape[0], 'train samples')
 print(X_test.shape[0], 'test samples' )<categorify>
start_time = time.perf_counter()
clf_mlp = MLPClassifier(hidden_layer_sizes=(784,), warm_start=True)
 print(clf_mlp)
 clf_mlp = clf_mlp.fit(features_train, target_train)
 target_predicted_mlp = clf_mlp.predict(features_test)
 print(time.perf_counter() - start_time, "seconds" )
num_classes = 10
y_train = to_categorical(y_train, num_classes)
 y_test = to_categorical(y_test, num_classes )<choose_model_class>
start_time = time.perf_counter()
scores_mlp = cross_val_score(clf_mlp, features_train, target_train, cv=10, n_jobs=-1, scoring='accuracy')
 print("Cross Validation Score for MLP",scores_mlp)
 print("Accuracy: %0.2f(+/- %0.2f)" %(scores_mlp.mean() , scores_mlp.std() * 2))
print(time.perf_counter() - start_time, "seconds" )
model = Sequential()
model.add(Conv2D(filters = 32, kernel_size =(5,5),padding = 'Same',
activation ='relu', input_shape =(img_rows,img_cols,1)))
 model.add(Conv2D(filters = 32, kernel_size =(5,5),padding = 'Same',
activation ='relu'))
model.add(MaxPool2D(pool_size=(2,2)))
 model.add(Dropout(0.25))
model.add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same',
activation ='relu'))
model.add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same',
activation ='relu'))
model.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))
 model.add(Dropout(0.25))
model.add(Flatten())
 model.add(Dense(256, activation = "relu"))
model.add(Dropout(0.5))
model.add(Dense(10, activation = "softmax"))<train_model>
digit_test['Label'] = pd.Series(target_predicted_mlp)
 digit_test['ImageId'] = pd.Series(range(1,28001))
optimizer = RMSprop(lr=0.001, rho=0.9, epsilon=1e-08, decay=0.0)
 model.compile(optimizer = optimizer , loss = "categorical_crossentropy", metrics=["accuracy"])
 learning_rate_reduction = ReduceLROnPlateau(monitor='val_loss',patience=3,factor=0.2,min_lr=0.00001)
 datagen = ImageDataGenerator(rotation_range=10,zoom_range = 0.1, width_shift_range=0.1, height_shift_range=0.1)
 hist = model.fit(x=datagen.flow(X_train,y_train, batch_size=128),
epochs = 20,
verbose = 2, steps_per_epoch=X_train.shape[0] // 128
,callbacks=[learning_rate_reduction] )<compute_test_metric>
digit_test.to_csv('submission_mlp.csv', columns=["ImageId","Label"], index=False )
score = model.evaluate(X_test, y_test, verbose=0)
 print('Test loss:', score[0])
 print('Test accuracy:', score[1] )<load_from_csv>
import keras as keras
from keras.utils.np_utils import to_categorical
from keras.models import Sequential
from keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPool2D
from keras.optimizers import RMSprop
from keras.preprocessing.image import ImageDataGenerator
from keras.callbacks import ReduceLROnPlateau
test = pd.read_csv('.. /input/Kannada-MNIST/test.csv' )<data_type_conversions>
X_train = features_train.values.reshape(-1,28,28,1)
 test = features_test.values.reshape(-1,28,28,1 )
X_test = test.values[:, 1:]
X_test = func_prep(X_test)
 X_test = X_test.reshape(X_test.shape[0], img_rows, img_cols, 1)
 X_test = X_test.astype('float32')
 X_test /= 255 
<predict_on_test>
Y_train = to_categorical(target_train, num_classes = 10 )
t = time.process_time()
preds = np.argmax(model.predict(X_test), axis=-1)
 elapsed_time = time.process_time() - t
print('elapsed_time: ', elapsed_time )<save_to_csv>
X_train, X_val, Y_train, Y_val = train_test_split(X_train, Y_train, test_size = 0.1, random_state=0 )
df = pd.DataFrame({'id': [ i for i in range(len(preds)) ], 'label': preds})
 df.to_csv('submission.csv', index=False )<import_modules>
clf_keras = Sequential()
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
import seaborn as sns
from sklearn.model_selection import train_test_split
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Conv2D, MaxPool2D, Flatten, Dropout, BatchNormalization
from tensorflow.keras.utils import to_categorical
from tensorflow.keras.callbacks import ReduceLROnPlateau, EarlyStopping
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras.optimizers import RMSprop, Adam, Adagrad, Adamax, Adadelta
from sklearn.metrics import classification_report, confusion_matrix<load_from_csv>
clf_keras.add(Conv2D(filters = 32, kernel_size =(5,5), padding = 'same', strides=1, activation ='relu',
input_shape =(28,28,1)))
 clf_keras.add(Conv2D(filters = 32, kernel_size =(5,5), padding = 'same', activation ='relu'))
clf_keras.add(keras.layers.BatchNormalization())
 clf_keras.add(keras.layers.MaxPooling2D(pool_size=(2,2)) )
train = pd.read_csv(".. /input/Kannada-MNIST/train.csv")
 test = pd.read_csv(".. /input/Kannada-MNIST/test.csv" )<prepare_x_and_y>
clf_keras.add(Conv2D(filters = 64, kernel_size =(3,3), strides=2, padding = 'same', activation ='relu'))
clf_keras.add(Conv2D(filters = 64, kernel_size =(3,3), padding = 'same', activation ='relu'))
clf_keras.add(keras.layers.BatchNormalization())
 clf_keras.add(keras.layers.MaxPooling2D(pool_size=(2,2)) )
train_X, train_y = train.drop(columns=['label']), train["label"]<prepare_x_and_y>
clf_keras.add(Dropout(rate = 0.5))
train_X = np.array(train_X)
 train_y = np.array(train_y )<drop_column>
clf_keras.add(Dense(10, activation = "softmax"))
testx = test.drop(columns=['id'] )<concatenate>
optimizer = keras.optimizers.Nadam(lr=0.002, beta_1=0.9, beta_2=0.999, epsilon=None, schedule_decay=0.004 )
testx = np.array(testx )<feature_engineering>
clf_keras.compile(optimizer = optimizer, loss = "categorical_crossentropy", metrics=["accuracy"] )
train_X = train_X / 255.0
testx = testx / 255.0<concatenate>
learning_rate_reduction = ReduceLROnPlateau(monitor='val_acc', factor=0.2, patience=3, min_lr=0.00001, verbose=1 )
train_y = to_categorical(train_y, num_classes = len(np.unique(train["label"])) )<split>
datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=10,
zoom_range = 0.10,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False)
 datagen.fit(X_train )
X_train, X_test, y_train, y_test = train_test_split(train_X, train_y, test_size=0.25, random_state=42, stratify= train_y, shuffle=True )<define_variables>
epochs = 30
epochs = 20
batch_size = 16<choose_model_class>
batch_size = 86
def create_model() :
model = Sequential()
model.add(Conv2D(filters = 32, kernel_size =(5,5),padding = 'Same', activation ='relu', input_shape =(28,28,1)))
 model.add(BatchNormalization())
 model.add(Conv2D(filters = 32, kernel_size =(5,5),padding = 'Same', activation ='relu'))
model.add(BatchNormalization())
 model.add(MaxPool2D(pool_size=(2,2)))
 model.add(Dropout(0.25))
model.add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same', activation ='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same', activation ='relu'))
model.add(BatchNormalization())
 model.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))
 model.add(Dropout(0.25))
model.add(Conv2D(filters = 128, kernel_size =(3,3),padding = 'Same', activation ='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(filters = 128, kernel_size =(3,3),padding = 'Same', activation ='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(filters = 128, kernel_size =(3,3),padding = 'Same', activation ='relu'))
model.add(BatchNormalization())
 model.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))
 model.add(Dropout(0.25))
model.add(Conv2D(filters = 256, kernel_size =(3,3),padding = 'Same', activation ='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(filters = 256, kernel_size =(3,3),padding = 'Same', activation ='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(filters = 256, kernel_size =(3,3),padding = 'Same', activation ='relu'))
model.add(BatchNormalization())
 model.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))
 model.add(Dropout(0.25))
model.add(Flatten())
 model.add(Dense(512, activation = "relu"))
model.add(BatchNormalization())
 model.add(Dropout(0.3))
model.add(Dense(10, activation = "softmax"))
return model<choose_model_class>
start_time = time.perf_counter()
history = clf_keras.fit_generator(datagen.flow(X_train,Y_train, batch_size=batch_size), epochs = epochs,
validation_data =(X_val,Y_val), verbose = 2,
steps_per_epoch=features_train.shape[0] // batch_size,
callbacks=[learning_rate_reduction],
use_multiprocessing=True)
 print(time.perf_counter() - start_time, "seconds" )
optimizer = Adamax()
model.compile(optimizer=optimizer, loss = 'categorical_crossentropy', metrics=['accuracy'])
 lr_reduction = ReduceLROnPlateau(monitor='val_acc',
patience=3,
verbose=1,
factor=0.5,
min_lr=0.00001)
 es = EarlyStopping(monitor='val_loss',
min_delta=0,
patience=5,
verbose=0, mode='auto')
<init_hyperparams>
results = clf_keras.predict(test)
 results = np.argmax(results,axis = 1)
 results = pd.Series(results,name="Label" )
datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.12,
height_shift_range=0.12,
horizontal_flip=False,
vertical_flip=False)
 datagen.fit(X_train )<train_model>
submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),results],axis = 1 )
<compute_test_metric><EOS> 
submission.to_csv("submission_keras.csv",index=False )