Kaggle competition: Digit Recognizer
train = pd.read_csv(f'{DATA_DIR}/train.csv' )<set_options>
warnings.filterwarnings('ignore' )
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
def get_img(path):
im_bgr = cv2.imread(path)
 im_rgb = im_bgr[:, :, ::-1]
return im_rgb<categorify>
train_df = pd.read_csv('.. /input/train.csv')
 test_df = pd.read_csv('.. /input/test.csv')
 print(train_df.shape)
 print(test_df.shape )
class CassavaDataset(Dataset):
def __init__(self, df, data_root, transforms=None, output_label=True):
super().__init__()
self.df = df.reset_index(drop=True ).copy()
self.transforms = transforms
self.data_root = data_root
self.output_label = output_label
def __len__(self):
return self.df.shape[0]
def __getitem__(self, index: int):
if self.output_label:
target = self.df.iloc[index]['label']
img = get_img(f"{self.data_root}/{self.df.loc[index]['image_id']}")
 if self.transforms:
img = self.transforms(image=img)['image']
if self.output_label:
return img, target
else:
return img<choose_model_class>
print('Missing values in training dataset : %d' %np.sum(train_df.isnull().sum()))
print('Missing values in testing dataset : %d' %np.sum(test_df.isnull().sum()))
class CassvaImgClassifierN(nn.Module):
def __init__(self, model_arch, n_class, pretrained=False):
super().__init__()
self.model = timm.create_model(model_arch, pretrained=pretrained)
 n_features = self.model.classifier.in_features
self.model.classifier = nn.Linear(n_features, n_class)
 def forward(self, x):
x = self.model(x)
 return x<choose_model_class>
train_df = train_df/255.0
test_df = test_df/255.0
train_df = train_df.values.reshape(-1,28,28,1)
 test_df = test_df.values.reshape(-1,28,28,1 )
class CassvaImgClassifier(nn.Module):
def __init__(self, model_arch, n_class, pretrained=False):
super().__init__()
self.n_class = n_class
self.model = timm.create_model(model_arch, pretrained=pretrained)
 if 'vit' in model_arch:
n_features = self.model.head.in_features
self.model.head = nn.Identity()
if 'eff' in model_arch:
n_features = self.model.classifier.in_features
self.model.classifier = nn.Identity()
if 'resnext' in model_arch:
n_features = self.model.fc.in_features
self.model.fc = nn.Identity()
self.layer_noise = nn.Linear(n_features, n_class)
 self.layer_pure = nn.Linear(n_features, n_class)
 def forward(self, x):
x = self.model(x)
 x1 = self.layer_noise(x)
 x2 = self.layer_pure(x)
 x1 = x1 + x2
return x1, x2<find_best_model_class>
train_y = to_categorical(train_label, num_classes = 10)
 print('one hot encoding vector of %d :' %train_label[3] , train_y[3] )
def inference_one_epoch(model, data_loader, device):
model.eval()
image_preds_all = []
pbar = tqdm(enumerate(data_loader), total=len(data_loader))
for step,(imgs)in pbar:
imgs = imgs.to(device ).float()
_, image_preds = model(imgs)
 image_preds_all += [torch.sigmoid(image_preds ).detach().cpu().numpy() ]
image_preds_all = np.concatenate(image_preds_all, axis=0)
 return image_preds_all<create_dataframe>
x_train,x_val,y_train,y_val = train_test_split(train_df,train_y,test_size = 0.2, random_state = 2)
def run_inference(fold, MODEL_DIR, model_arch, epoch_list):
print(f'Inference fold {fold} started')
 test = pd.DataFrame()
test['image_id'] = list(os.listdir(f'{DATA_DIR}/test_images/'))
if 'vit' in model_arch:
T = get_inference_transforms_384()
else:
T = get_inference_transforms()
test_ds = CassavaDataset(
 test, f'{DATA_DIR}/test_images/',
transforms=T, output_label=False)
 tst_loader = torch.utils.data.DataLoader(
 test_ds,
batch_size=CFG['valid_bs'],
num_workers=CFG['num_workers'],
shuffle=False,
pin_memory=False, 
)
 device = torch.device(CFG['device'])
 model = CassvaImgClassifier(model_arch, train.label.nunique() ).to(device)
 tst_preds = []
for i, epoch in enumerate(epoch_list[fold][:1]):
model.load_state_dict(torch.load(f'{MODEL_DIR}/{model_arch}_fold_{fold}_{epoch}'))
with torch.no_grad() :
for _ in range(CFG['tta']):
tst_image_preds = inference_one_epoch(model, tst_loader, device)
 tst_preds += [(1/CFG['tta'])*tst_image_preds]
tst_preds = np.sum(tst_preds, axis=0)
 del model
torch.cuda.empty_cache()
return tst_preds<create_dataframe>
dataGenerator = ImageDataGenerator(rotation_range = 10,
width_shift_range = 0.1,
height_shift_range = 0.1,
zoom_range = 0.1)
 dataGenerator.fit(x_train )
def run_inferenceN(fold, MODEL_DIR, model_arch):
print(f'Inference fold {fold} started')
 test = pd.DataFrame()
test['image_id'] = list(os.listdir(f'{DATA_DIR}/test_images/'))
test_ds = CassavaDataset(
 test, f'{DATA_DIR}/test_images/',
transforms=get_inference_transforms() , output_label=False)
 tst_loader = torch.utils.data.DataLoader(
 test_ds,
batch_size=CFG['valid_bs'],
num_workers=CFG['num_workers'],
shuffle=False,
pin_memory=False, 
)
 device = torch.device(CFG['device'])
 model = CassvaImgClassifierN(model_arch, train.label.nunique() ).to(device)
 tst_preds = []
for i, epoch in enumerate(CFG['used_epochs']):
model.load_state_dict(
 torch.load(f'{MODEL_DIR}/{model_arch}_fold_{fold}_{epoch}'))
with torch.no_grad() :
for _ in range(CFG['tta']):
tst_image_preds = inference_one_epochN(model, tst_loader, device)
 tst_preds += [(1/ CFG['tta'])*tst_image_preds]
tst_preds = np.mean(tst_preds, axis=0)
 del model
torch.cuda.empty_cache()
return tst_preds<define_variables>
ExampleImg = train_df[10][:,:,0]
def translate(img,x,y):
transMat = np.float32([[1,0,x],[0,1,y]])
 shifted = cv2.warpAffine(img,transMat,(img.shape[1],img.shape[0]))
return shifted
def rotate(img,angle): 
(h,w)= img.shape[:2]
center =(w/2,h/2)
 rotatMat = cv2.getRotationMatrix2D(center,angle,1)
 rotated = cv2.warpAffine(img,rotatMat,(w,h))
return rotated
def zoom(img,scale): 
(oh,ow)= img.shape[:2]
zoomed = cv2.resize(img, None, fx = scale, fy = scale)
(nh,nw)= zoomed.shape[:2]
extend1 = int(( oh - nh)/ 2.0 + 0.5)
 extend2 = oh -(nh+extend1)
 zoomed = cv2.copyMakeBorder(zoomed,extend1,extend2,extend1,extend2,cv2.BORDER_CONSTANT,0)
 return zoomed
shifted = translate(ExampleImg,0,3)
 rotated = rotate(ExampleImg,10)
 zoomed = zoom(ExampleImg,0.8)
 plt.figure(figsize =(10,20))
plt.subplot(1,4,1)
 plt.imshow(ExampleImg,cmap = plt.cm.binary)
 plt.title('Original')
 plt.subplot(1,4,2)
 plt.imshow(rotated,cmap = plt.cm.binary)
 plt.title('rotated: 10 deg')
 plt.subplot(1,4,3)
 plt.imshow(shifted,cmap = plt.cm.binary)
 plt.title('shifted: y-3')
 plt.subplot(1,4,4)
 plt.imshow(zoomed,cmap = plt.cm.binary)
 plt.title('zoomed: 0.8 scale')
 plt.show()
preds0 = run_inference(0, MODEL_DIR_0, 'vit_base_patch16_384', EPOCHS0)
 preds1 = run_inference(1, MODEL_DIR_1, 'vit_base_patch16_384', EPOCHS0)
 preds2 = run_inference(2, MODEL_DIR_2, 'vit_base_patch16_384', EPOCHS0)
 preds3 = run_inference(3, MODEL_DIR_3, 'vit_base_patch16_384', EPOCHS0)
 preds4 = run_inference(4, MODEL_DIR_4, 'vit_base_patch16_384', EPOCHS0)
 PRED0 =(preds0 + preds1 + preds2 + preds3 + preds4)/5<define_variables>
def build_CNN(input_shape, output_units = 10):
input_layer = keras.layers.Input(input_shape, name = "input_layer")
 x = Conv2D(filters = 64, kernel_size =(3,3), padding = 'same' )(input_layer)
 x = BatchNormalization()(x)
 x = Activation('relu' )(x)
 x = Conv2D(filters = 64, kernel_size =(3,3), padding = 'same' )(x)
 x = BatchNormalization()(x)
 x = Activation('relu' )(x)
 x = MaxPooling2D(pool_size=(2,2))(x)
 x = Dropout(0.25 )(x)
 x = Conv2D(filters = 32, kernel_size =(3,3), padding = 'same' )(x)
 x = BatchNormalization()(x)
 x = Activation('relu' )(x)
 x = Conv2D(filters = 32, kernel_size =(3,3), padding = 'same' )(x)
 x = BatchNormalization()(x)
 x = Activation('relu' )(x)
 x = MaxPooling2D(pool_size=(2,2))(x)
 x = Dropout(0.25 )(x)
 x = Flatten()(x)
 x = Dense(units = 512 )(x)
 x = BatchNormalization()(x)
 x = Activation('relu' )(x)
 x = Dense(units = 256 )(x)
 x = BatchNormalization()(x)
 x = Activation('relu' )(x)
 x = Dropout(0.25 )(x)
 x = Dense(units = 128 )(x)
 x = BatchNormalization()(x)
 x = Activation('relu' )(x)
 x = Dense(units = 64, activation = 'relu' )(x)
 output_layer = Dense(units = output_units, activation = 'softmax', name = "output_layer" )(x)
 model = keras.models.Model(inputs = [input_layer], outputs = [output_layer])
 return model
preds0 = run_inference(0, MODEL_DIR_01, 'tf_efficientnet_b4_ns', EPOCHS1)
 preds1 = run_inference(1, MODEL_DIR_11, 'tf_efficientnet_b4_ns', EPOCHS1)
 preds2 = run_inference(2, MODEL_DIR_21, 'tf_efficientnet_b4_ns', EPOCHS1)
 preds3 = run_inference(3, MODEL_DIR_31, 'tf_efficientnet_b4_ns', EPOCHS1)
 preds4 = run_inference(4, MODEL_DIR_41, 'tf_efficientnet_b4_ns', EPOCHS1)
 PRED1 =(preds0 + preds1 + preds2 + preds3 + preds4)/5<define_variables>
batch_size = 128
epochs = 300
momentum = 0.95
lr = 5e-4
ES = keras.callbacks.EarlyStopping(monitor = "val_loss",
patience = 10,
verbose = 1)
 model = build_CNN(x_train.shape[1:],10)
 optimizer = keras.optimizers.SGD(lr = lr, momentum = 0.95)
 model.compile(optimizer = optimizer, loss = 'categorical_crossentropy', metrics = ['accuracy'])
 model.summary()
history = model.fit_generator(dataGenerator.flow(x_train, y_train, batch_size = batch_size),
epochs = epochs, verbose = 2, validation_data =(x_val,y_val),
steps_per_epoch = x_train.shape[0]/batch_size )
preds0 = run_inference(0, MODEL_DIR_02, 'seresnext50_32x4d', EPOCHS2)
 preds1 = run_inference(1, MODEL_DIR_12, 'seresnext50_32x4d', EPOCHS2)
 preds2 = run_inference(2, MODEL_DIR_22, 'seresnext50_32x4d', EPOCHS2)
 preds3 = run_inference(3, MODEL_DIR_32, 'seresnext50_32x4d', EPOCHS2)
 preds4 = run_inference(4, MODEL_DIR_42, 'seresnext50_32x4d', EPOCHS2)
 PRED2 =(preds0 + preds1 + preds2 + preds3 + preds4)/5<define_variables>
final_prediction = model.predict(test_df)
 final_prediction = np.argmax(final_prediction, axis = 1 )
preds0 = run_inferenceN(0, MODEL_DIR_03, 'tf_efficientnet_b4_ns',)
 preds1 = run_inferenceN(1, MODEL_DIR_13, 'tf_efficientnet_b4_ns',)
 preds2 = run_inferenceN(2, MODEL_DIR_23, 'tf_efficientnet_b4_ns',)
 preds3 = run_inferenceN(3, MODEL_DIR_33, 'tf_efficientnet_b4_ns',)
 preds4 = run_inferenceN(4, MODEL_DIR_43, 'tf_efficientnet_b4_ns')
 PRED3 =(preds0 + preds1 + preds2 + preds3 + preds4)/5<define_variables>
submission = pd.DataFrame({'ImageId':np.arange(1,final_prediction.shape[0]+1,1),'Label':final_prediction})
 submission.to_csv('submission_v5.csv',index = False )