Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<define_variables>
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from keras.utils import to_categorical
from keras.preprocessing.image import ImageDataGenerator
mod_path = '.. /input/notebook43a3eaff7d-1/'
with open(mod_path+'params.yaml')as file:
P = yaml.load(file, Loader=yaml.FullLoader)
 pprint.pprint(P)
 THRESHOLD = 0.38
WINDOW = 1024
MIN_OVERLAP = 300
NEW_SIZE = P['DIM']
SUBMISSION_MODE = 'FULL'
CHECKSUM = True 
<load_pretrained>
np.random.seed(1)
 X_raw = pd.read_csv(".. /input/digit-recognizer/train.csv")
 X_test_raw = pd.read_csv(".. /input/digit-recognizer/test.csv")
 y = X_raw["label"]
X = X_raw.drop(labels = ["label"],axis = 1)
 X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.20, random_state=1)
 WIDTH=28
HEIGHT=28
NUM_CLASSES=10
X_train = X_train.values
y_train = y_train.values
X_valid = X_valid.values
y_valid = y_valid.values
y_train_oh = to_categorical(y_train, num_classes = NUM_CLASSES)
 y_valid_oh = to_categorical(y_valid, num_classes = NUM_CLASSES)
 X_test = X_test_raw.values
with open(mod_path + 'metrics.json')as json_file:
M = json.load(json_file)
 print('Model run datetime: '+M['datetime'])
 print('OOF val_dice_coe: ' + str(M['oof_dice_coe']))<install_modules>
def reshape_channel(X):
return X.reshape(-1,HEIGHT,WIDTH,1)
 def preprocessData(X):
X = reshape_channel(X)
 return X
! pip install.. /input/kerasapplications/keras-team-keras-applications-3b180cb -f./ --no-index -q
! pip install.. /input/efficientnet/efficientnet-1.1.0/ -f./ --no-index -q 
<categorify>
optimizer = Adam(learning_rate=0.0001, beta_1=0.9, beta_2=0.9999, amsgrad=True)
 model = Sequential()
model.add(Conv2D(filters = 64, kernel_size =(5,5),padding = 'same',activation ='relu',input_shape =(HEIGHT,WIDTH,1)))
 model.add(Conv2D(filters = 64, kernel_size =(5,5),padding = 'same',activation ='relu'))
model.add(Conv2D(filters = 64, kernel_size =(5,5),padding = 'same',activation ='relu'))
model.add(MaxPool2D(pool_size=(5,5),padding='same'))
model.add(Dropout(0.2))
model.add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'same',activation ='relu'))
model.add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'same',activation ='relu'))
model.add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'same',activation ='relu'))
model.add(MaxPool2D(pool_size=(3,3),padding='same'))
model.add(Dropout(0.2))
model.add(Flatten())
 model.add(Dense(256, activation = "relu"))
model.add(Dense(256, activation = "relu"))
model.add(Dropout(0.2))
model.add(Dense(NUM_CLASSES, activation = "softmax"))
model.compile(optimizer ='adam', loss = "categorical_crossentropy", metrics=["accuracy"])
 model.summary()
def rle_encode_less_memory(img):
pixels = img.T.flatten()
pixels[0] = 0
pixels[-1] = 0
runs = np.where(pixels[1:] != pixels[:-1])[0] + 2
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs)
 def make_grid(shape, window=256, min_overlap=32):

 x, y = shape
nx = x //(window - min_overlap)+ 1
x1 = np.linspace(0, x, num=nx, endpoint=False, dtype=np.int64)
 x1[-1] = x - window
x2 =(x1 + window ).clip(0, x)
 ny = y //(window - min_overlap)+ 1
y1 = np.linspace(0, y, num=ny, endpoint=False, dtype=np.int64)
 y1[-1] = y - window
y2 =(y1 + window ).clip(0, y)
 slices = np.zeros(( nx,ny, 4), dtype=np.int64)
 for i in range(nx):
for j in range(ny):
slices[i,j] = x1[i], x2[i], y1[j], y2[j]
return slices.reshape(nx*ny,4 )<load_pretrained>
datagen_with_aug = ImageDataGenerator(
 rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1,
rescale=1./255, 
)
 datagen_with_aug.fit(preprocessData(X_train))
datagen_no_aug = ImageDataGenerator()
train_generator = datagen_with_aug.flow(preprocessData(X_train), y_train_oh, batch_size=128)
 valid_generator = datagen_with_aug.flow(preprocessData(X_valid), y_valid_oh,batch_size=128 )
identity = rasterio.Affine(1, 0, 0, 0, 1, 0)
 fold_models = []
for fold_model_path in glob.glob(mod_path+'*.h5'):
fold_models.append(tf.keras.models.load_model(fold_model_path,compile = False))
print(len(fold_models))<prepare_x_and_y>
keras.initializers.RandomNormal(mean=0.0, stddev=0.1, seed=1)
 hist = model.fit_generator(train_generator,validation_data=valid_generator,callbacks=[lrate],epochs=n_epoch,steps_per_epoch=(X_train.shape[0]/128)*2)
AUTO = tf.data.experimental.AUTOTUNE
image_feature = {
'image': tf.io.FixedLenFeature([], tf.string),
'x1': tf.io.FixedLenFeature([], tf.int64),
'y1': tf.io.FixedLenFeature([], tf.int64)
 }
def _parse_image(example_proto):
example = tf.io.parse_single_example(example_proto, image_feature)
 image = tf.reshape(tf.io.decode_raw(example['image'],out_type=np.dtype('uint8')) ,(P['DIM'],P['DIM'], 3))
return image, example['x1'], example['y1']
def load_dataset(filenames, ordered=True):
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames)
 dataset = dataset.with_options(ignore_order)
 dataset = dataset.map(_parse_image)
 return dataset
def get_dataset(FILENAME):
dataset = load_dataset(FILENAME)
 dataset = dataset.batch(64)
 dataset = dataset.prefetch(AUTO)
 return dataset<define_variables>
y_pred = model.predict(preprocessData(X_test))
y_pred = np.argmax(y_pred,axis = 1)
 submission = pd.DataFrame({'ImageId':range(1,28001),'Label':y_pred})
 submission.to_csv('submission.csv',index=False )