Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<import_modules>
%matplotlib inline
HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,
Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,
IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, RandomResizedCrop,
IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout,
ShiftScaleRotate, CenterCrop, Resize 
)
 HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,
Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,
IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, RandomResizedCrop,
IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout,
ShiftScaleRotate, CenterCrop, Resize 
)
 def get_train_transforms() :
return Compose([
RandomResizedCrop(CFG['img_size'], CFG['img_size']),
Transpose(p=0.5),
HorizontalFlip(p=0.5),
VerticalFlip(p=0.5),
ShiftScaleRotate(p=0.5),
HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),
RandomBrightnessContrast(brightness_limit=(-0.1, 0.1), contrast_limit=(-0.1, 0.1), p=0.5),
Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),
CoarseDropout(p=0.5),
Cutout(p=0.5),
ToTensorV2(p=1.0),
], p=1.)
train = pd.read_csv('.. /input/cassava-leaf-disease-classification/train.csv')
 def get_valid_transforms() :
return Compose([
CenterCrop(CFG['img_size'], CFG['img_size'], p=1.) ,
Resize(CFG['img_size'], CFG['img_size']),
Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),
ToTensorV2(p=1.0),
], p=1.)
def rand_bbox(size, lam):
W = size[0]
H = size[1]
cut_rat = np.sqrt(1.- lam)
 cut_w = np.int(W * cut_rat)
 cut_h = np.int(H * cut_rat)
 cx = np.random.randint(W)
 cy = np.random.randint(H)
 bbx1 = np.clip(cx - cut_w // 2, 0, W)
 bby1 = np.clip(cy - cut_h // 2, 0, H)
 bbx2 = np.clip(cx + cut_w // 2, 0, W)
 bby2 = np.clip(cy + cut_h // 2, 0, H)
 return bbx1, bby1, bbx2, bby2
def get_inference_transforms() :
return Compose([
RandomResizedCrop(CFG['img_size'], CFG['img_size']),
Transpose(p=0.5),
HorizontalFlip(p=0.5),
VerticalFlip(p=0.5),
HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),
RandomBrightnessContrast(brightness_limit=(-0.1, 0.1), contrast_limit=(-0.1, 0.1), p=0.5),
Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),
ToTensorV2(p=1.0),
], p=1.)
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
CFG = {
'fold_num': 5,
'seed': 719,
'model_arch': 'tf_efficientnet_b4_ns',
'img_size': 512,
'epochs': 10,
'train_bs': 32,
'valid_bs': 32,
'lr': 1e-4,
'num_workers': 4,
'accum_iter': 1,
'verbose_step': 1,
'device': 'cuda:0',
'tta': 3,
'used_epochs': [25,26,27,28,29],
'weights': [1,1,1,1]
}
class CassavaNet(nn.Module):
def __init__(self):
super().__init__()
backbone = timm.create_model('tf_efficientnet_b4_ns', pretrained=False)
 n_features = backbone.classifier.in_features
self.backbone = nn.Sequential(*backbone.children())[:-3]
self.classifier = nn.Linear(n_features, 5)
 self.pool = nn.AdaptiveAvgPool2d(( 1, 1))
def forward_features(self, x):
x = self.backbone(x)
 return x
def forward(self, x):
feats = self.forward_features(x)
 x = self.pool(feats ).view(x.size(0), -1)
 x = self.classifier(x)
 return x, feats
class CassavaDataset(Dataset):
def __init__(self, df, data_root,
transforms=None,
output_label=True,
one_hot_label=False,
do_fmix=False,
fmix_params={
'alpha': 1.,
'decay_power': 3.,
'shape':(CFG['img_size'], CFG['img_size']),
'max_soft': True,
'reformulate': False
},
do_cutmix=False,
cutmix_params={
'alpha': 1,
} 
):
super().__init__()
self.df = df.reset_index(drop=True ).copy()
self.transforms = transforms
self.data_root = data_root
self.do_fmix = do_fmix
self.fmix_params = fmix_params
self.do_cutmix = do_cutmix
self.cutmix_params = cutmix_params
self.output_label = output_label
self.one_hot_label = one_hot_label
if output_label == True:
self.labels = self.df['label'].values
if one_hot_label is True:
self.labels = np.eye(self.df['label'].max() + 1)[self.labels]
def __len__(self):
return self.df.shape[0]
def __getitem__(self, index: int):
if self.output_label:
target = self.labels[index]
img = get_img("{}/{}".format(self.data_root, self.df.loc[index]['image_id']))
if self.transforms:
img = self.transforms(image=img)['image']
if self.do_fmix and np.random.uniform(0., 1., size=1)[0] > 0.5:
with torch.no_grad() :
lam = np.clip(np.random.beta(self.fmix_params['alpha'], self.fmix_params['alpha']), 0.6, 0.7)
 mask = make_low_freq_image(self.fmix_params['decay_power'], self.fmix_params['shape'])
 mask = binarise_mask(mask, lam, self.fmix_params['shape'], self.fmix_params['max_soft'])
 fmix_ix = np.random.choice(self.df.index, size=1)[0]
fmix_img = get_img("{}/{}".format(self.data_root, self.df.iloc[fmix_ix]['image_id']))
if self.transforms:
fmix_img = self.transforms(image=fmix_img)['image']
mask_torch = torch.from_numpy(mask)
 img = mask_torch * img +(1.- mask_torch)* fmix_img
rate = mask.sum() / CFG['img_size'] / CFG['img_size']
target = rate * target +(1.- rate)* self.labels[fmix_ix]
if self.do_cutmix and np.random.uniform(0., 1., size=1)[0] > 0.5:
with torch.no_grad() :
cmix_ix = np.random.choice(self.df.index, size=1)[0]
cmix_img = get_img("{}/{}".format(self.data_root, self.df.iloc[cmix_ix]['image_id']))
if self.transforms:
cmix_img = self.transforms(image=cmix_img)['image']
lam = np.clip(np.random.beta(self.cutmix_params['alpha'], self.cutmix_params['alpha']), 0.3, 0.4)
 bbx1, bby1, bbx2, bby2 = rand_bbox(( CFG['img_size'], CFG['img_size']), lam)
 img[:, bbx1:bbx2, bby1:bby2] = cmix_img[:, bbx1:bbx2, bby1:bby2]
rate = 1 -(( bbx2 - bbx1)*(bby2 - bby1)/(CFG['img_size'] * CFG['img_size']))
target = rate * target +(1.- rate)* self.labels[cmix_ix]
if self.output_label == True:
return img, target
else:
return img
def inference_one_epoch(model, data_loader, device):
model.eval()
image_preds_all = []
pbar = tqdm(enumerate(data_loader), total=len(data_loader))
for step,(imgs)in pbar:
imgs = imgs.to(device ).float()
image_preds= model(imgs)
 image_preds_all += [torch.softmax(image_preds, 1 ).detach().cpu().numpy() ]
image_preds_all = np.concatenate(image_preds_all, axis=0)
 return image_preds_all
def get_img(path):
im_bgr = cv2.imread(path)
 im_rgb = im_bgr[:, :, ::-1]
return im_rgb
class CassvaImgClassifier(nn.Module):
def __init__(self, model_arch, n_class, pretrained=False):
super().__init__()
self.model = timm.create_model(model_arch, pretrained=pretrained)
 n_features = self.model.classifier.in_features
self.model.classifier = nn.Linear(n_features, n_class)

def forward(self, x):
x = self.model(x)
 return x
if __name__ == '__main__':
seed_everything(CFG['seed'])
 folds = StratifiedKFold(n_splits=CFG['fold_num'] ).split(np.arange(train.shape[0]), train.label.values)
 for fold,(trn_idx, val_idx)in enumerate(folds):
if fold > 0:
break
print('Inference fold {} started'.format(fold))
valid_ = train.loc[val_idx, :].reset_index(drop=True)
 valid_ds = CassavaDataset(valid_, '.. /input/cassava-leaf-disease-classification/train_images/',
transforms=get_inference_transforms() , output_label=False)
 test = pd.DataFrame()
test['image_id'] = list(os.listdir('.. /input/cassava-leaf-disease-classification/test_images/'))
test_ds = CassavaDataset(test, '.. /input/cassava-leaf-disease-classification/test_images/',
transforms=get_inference_transforms() , output_label=False)
 tst_loader = torch.utils.data.DataLoader(
 test_ds,
batch_size=CFG['valid_bs'],
num_workers=CFG['num_workers'],
shuffle=False,
pin_memory=False, 
)
 device = torch.device(CFG['device'])
 model = CassvaImgClassifier(CFG['model_arch'], train.label.nunique() ).to(device)
 tst_preds = []
for i in [1]:
for epoch in [1,2,3,4,5,6,7,8,9]:
ckpt=torch.load('.. /input/model-comined/tf_efficientnet_b4_ns_fold_{}_{}.pth'.format(i,epoch),map_location={'cuda:1':'cuda:0'})
 ckpt_model_dict = OrderedDict([(k[7:],v)if 'module.' in k else(k,v)for k, v in ckpt.items() ])
 model.load_state_dict(ckpt_model_dict)
 with torch.no_grad() :
tst_preds += [inference_one_epoch(model, tst_loader,device)]
tst_preds = np.mean(tst_preds, axis=0)
 del model
torch.cuda.empty_cache()
test['label'] = np.argmax(tst_preds, axis=1)
 test.head()
test.to_csv('submission.csv', index=False)
 print(test)
<import_modules>
train = pd.read_csv('.. /input/digit-recognizer/train.csv')
 test = pd.read_csv('.. /input/digit-recognizer/test.csv' )
package_path = '.. /input/pytorch-image-models/pytorch-image-models-master'
train = pd.read_csv('.. /input/cassava-leaf-disease-classification/train.csv')
 submission = pd.read_csv('.. /input/cassava-leaf-disease-classification/sample_submission.csv' )<init_hyperparams>
Y_train=train['label']
X_train = train.drop(labels = ["label"],axis = 1)
 Y_train.value_counts()
CFG = {
'normalize_mean':[0.42984136, 0.49624753, 0.3129598],
'normalize_std':[0.21417203, 0.21910103, 0.19542212],
'device': 'cuda:0',
'fold_num': 5,
'seed': 42,
'valid_bs': 32,
'num_workers': 4,
'model_arch': ['tf_efficientnet_b4_ns',
'tf_efficientnet_b4_ns',
'tf_efficientnet_b4_ns',
'tf_efficientnet_b4_ns',
'tf_efficientnet_b4_ns',
],
'img_size': 512,
'tta': 3,
'fold_list':[0],
'used_epochs': [".. /input/train3012/tf_efficientnet_b4_ns_3012_fold_0_14",
".. /input/train3012/tf_efficientnet_b4_ns_3012_fold_1_14",
".. /input/train3012/tf_efficientnet_b4_ns_3012_fold_2_14",
".. /input/train3012/tf_efficientnet_b4_ns_3012_fold_3_9",
".. /input/train3012/tf_efficientnet_b4_ns_3012_fold_4_12"],
'weights': [1,1,1,1,1],
}<set_options>
X_train = X_train / 255
test = test / 255
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
torch.cuda.manual_seed_all(seed)
 def get_img(path):
im_bgr = cv2.imread(path)
 im_rgb = im_bgr[:, :, ::-1]
return im_rgb<categorify>
print("The shape of the labels before One Hot Encoding",Y_train.shape)
 Y_train = to_categorical(Y_train, num_classes = 10)
 print("The shape of the labels after One Hot Encoding",Y_train.shape )
class CassavaDataset(Dataset):
def __init__(
 self, df, data_root, transforms=None, output_label=True 
):
super().__init__()
self.df = df.reset_index(drop=True ).copy()
self.transforms = transforms
self.data_root = data_root
self.output_label = output_label
def __len__(self):
return self.df.shape[0]
def __getitem__(self, index: int):
if self.output_label:
target = self.df.iloc[index]['label']
path = "{}/{}".format(self.data_root, self.df.iloc[index]['image_id'])
 img = get_img(path)
 if self.transforms:
img = self.transforms(image=img)['image']
if self.output_label == True:
return img, target
else:
return img<normalization>
random_seed = 2
X_train, X_val, Y_train, Y_val = train_test_split(X_train, Y_train, test_size = 0.3, random_state=random_seed )
HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,
Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,
IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, RandomResizedCrop,
IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout, ShiftScaleRotate, CenterCrop, Resize,
RandomCrop 
)
 def get_inference_transforms() :
return Compose([
RandomCrop(512, 512),
Transpose(p=0.5),
HorizontalFlip(p=0.5),
VerticalFlip(p=0.5),
HueSaturationValue(hue_shift_limit=10, sat_shift_limit=10, val_shift_limit=10, p=0.5),
RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5),
Normalize(mean=CFG["normalize_mean"], std=CFG["normalize_std"], max_pixel_value=255.0, p=1.0),
ToTensorV2(p=1.0),
], p=1.)
def get_inference_transforms_vit384() :
return Compose([
RandomCrop(384, 384),
Transpose(p=0.5),
HorizontalFlip(p=0.5),
VerticalFlip(p=0.5),
HueSaturationValue(hue_shift_limit=10, sat_shift_limit=10, val_shift_limit=10, p=0.5),
RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5),
Normalize(mean=CFG["normalize_mean"], std=CFG["normalize_std"], max_pixel_value=255.0, p=1.0),
ToTensorV2(p=1.0),
], p=1.) 
<choose_model_class>
datagen = ImageDataGenerator(zoom_range = 0.1, width_shift_range = 0.1, height_shift_range = 0.1, rotation_range = 10 )
class CassvaImgClassifier(nn.Module):
def __init__(self, model_arch, n_class, pretrained=False):
super().__init__()
self.model = timm.create_model(model_arch, pretrained=pretrained, num_classes=5)
 def forward(self, x):
x = self.model(x)
 return x<create_dataframe>
model = Sequential()
model.add(Conv2D(filters = 32, kernel_size =(3, 3), activation = 'relu', input_shape =(28, 28, 1)))
 model.add(BatchNormalization())
 model.add(Conv2D(filters = 32, kernel_size =(3, 3), activation = 'relu'))
model.add(BatchNormalization())
 model.add(Conv2D(filters = 32, kernel_size =(5, 5), activation = 'relu'))
model.add(BatchNormalization())
 model.add(MaxPool2D(strides =(2,2)))
 model.add(Dropout(0.25))
model.add(Conv2D(filters = 64, kernel_size =(3, 3), activation = 'relu'))
model.add(BatchNormalization())
 model.add(Conv2D(filters = 64, kernel_size =(3, 3), activation = 'relu'))
model.add(BatchNormalization())
 model.add(Conv2D(filters = 64, kernel_size =(5, 5), activation = 'relu'))
model.add(BatchNormalization())
 model.add(MaxPool2D(strides =(2,2)))
 model.add(Dropout(0.25))
model.add(Flatten())
 model.add(Dense(512, activation = 'relu'))
model.add(Dropout(0.5))
model.add(Dense(1024, activation = 'relu'))
model.add(Dropout(0.5))
model.add(Dense(10, activation = 'softmax'))
seed_everything(CFG['seed'])
 tst_preds = []
device = torch.device(CFG['device'])
 test = pd.DataFrame()
test['image_id'] = list(os.listdir('.. /input/cassava-leaf-disease-classification/test_images/'))
for i, sub_model in enumerate(CFG['used_epochs']):
if "vit" not in sub_model:
test_ds = CassavaDataset(test, '.. /input/cassava-leaf-disease-classification/test_images/', transforms=get_inference_transforms() , output_label=False)
 else:
test_ds = CassavaDataset(test, '.. /input/cassava-leaf-disease-classification/test_images/', transforms=get_inference_transforms_vit384() , output_label=False)
 tst_loader = torch.utils.data.DataLoader(
 test_ds,
batch_size=CFG['valid_bs'],
num_workers=CFG['num_workers'],
shuffle=False,
pin_memory=True, 
)
 model = CassvaImgClassifier(CFG['model_arch'][i], train.label.nunique() ).to(device)
 model.load_state_dict(torch.load(sub_model, map_location=CFG['device']))
print(sub_model)
 with torch.no_grad() :
for tta_ in range(CFG['tta']):
tst_preds += [CFG['weights'][i]/sum(CFG['weights'])/CFG['tta']*inference_one_epoch(model, tst_loader, device)]
tst_preds = np.sum(tst_preds, axis=0)
 del model
torch.cuda.empty_cache()<save_to_csv>
model.compile(optimizer='adam',metrics=['accuracy'],loss='categorical_crossentropy' )
test['label'] = np.argmax(tst_preds, axis=1)
 test.to_csv('submission.csv', index=False )<install_modules>
reduction_lr = ReduceLROnPlateau(monitor='val_accuracy',patience=2, verbose=1, factor=0.2, min_lr=0.00001 )
!mkdir -p /tmp/pip/cache/
!cp.. /input/omegaconf/PyYAML-5.4b2-cp38-cp38-manylinux1_x86_64.whl /tmp/pip/cache/
!cp.. /input/omegaconf/omegaconf-2.0.5-py3-none-any.whl /tmp/pip/cache/
!cp.. /input/omegaconf/typing_extensions-3.7.4.3-py3-none-any.whl /tmp/pip/cache/
!pip install --no-index --find-links /tmp/pip/cache/ omegaconf > /dev/null<set_options>
hist = model.fit_generator(datagen.flow(X_train,Y_train,batch_size=32),epochs=20,validation_data =(X_val,Y_val),callbacks=[reduction_lr] )
sys.path.append('.. /input/timm-pytorch-image-models/pytorch-image-models-master')
 sys.path.append(".. /input/cleanlab/")
 warnings.filterwarnings('ignore')
<set_options>
final_loss, final_acc = model.evaluate(X_val, Y_val, verbose=0)
 print("Final loss: {0:.4f}, final accuracy: {1:.4f}".format(final_loss, final_acc))
mean, std =(0.485, 0.456, 0.406),(0.229, 0.224, 0.225)
 def get_transforms(img_size=(512, 512)) :
transformations = Compose([
PadIfNeeded(min_height=img_size[0], min_width=img_size[1]),
CenterCrop(img_size[0], img_size[1]),
Normalize(mean=mean, std=std, max_pixel_value=255.0, p=1.0),
ToTensorV2(p=1.0),
], p=1.0)
 return transformations<choose_model_class>
y_pred = model.predict(X_val, batch_size = 64)
 y_pred = np.argmax(y_pred,axis = 1)
 y_pred = pd.Series(y_pred,name="Label")
 y_pred
def create_model(model_name: str,
pretrained: bool,
num_classes: int,
in_chans: int):
model = timm.create_model(model_name=model_name,
pretrained=pretrained,
num_classes=num_classes,
in_chans=in_chans)
 return model<load_pretrained>
y_pred1 = model.predict(test, batch_size = 64)
 y_pred1 = np.argmax(y_pred1,axis = 1)
 y_pred1 = pd.Series(y_pred1,name="Label")
 y_pred1
<find_best_params><EOS> 
submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),y_pred1],axis = 1)
 submission.to_csv("submission.csv",index=False )