Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<prepare_x_and_y>
%matplotlib inline
def rand_bbox(size, lam):
W = size[0]
H = size[1]
cut_rat = np.sqrt(1.- lam)
 cut_w = np.int(W * cut_rat)
 cut_h = np.int(H * cut_rat)
 cx = np.random.randint(W)
 cy = np.random.randint(H)
 bbx1 = np.clip(cx - cut_w // 2, 0, W)
 bby1 = np.clip(cy - cut_h // 2, 0, H)
 bbx2 = np.clip(cx + cut_w // 2, 0, W)
 bby2 = np.clip(cy + cut_h // 2, 0, H)
 return bbx1, bby1, bbx2, bby2
class CassavaDataset(Dataset):
def __init__(self, df, data_root,
transforms=None,
output_label=True, 
):
super().__init__()
self.df = df.reset_index(drop=True ).copy()
self.transforms = transforms
self.data_root = data_root
self.output_label = output_label
if self.output_label == True:
self.labels = self.df['label'].values
def __len__(self):
return self.df.shape[0]
def __getitem__(self, index: int):
if self.output_label:
target = self.labels[index]
img = get_img("{}/{}".format(self.data_root, self.df.loc[index]['image_id']))
if self.transforms:
img = self.transforms(image=img)['image']
if self.output_label == True:
return img, target
else:
return img<categorify>
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,
Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,
IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, RandomResizedCrop,
IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout, ShiftScaleRotate, CenterCrop, Resize, Rotate,
ShiftScaleRotate, CenterCrop, Resize, Rotate, RandomShadow, RandomSizedBBoxSafeCrop,
ChannelShuffle, MotionBlur 
)
 def get_train_transforms() :
return Compose([
RandomResizedCrop(CFG['img_size1'], CFG['img_size1']),
Transpose(p=0.5),
HorizontalFlip(p=0.5),
VerticalFlip(p=0.5),
ShiftScaleRotate(p=0.5),
HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),
RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5),
Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),
CoarseDropout(p=0.5),
Cutout(p=0.5),
ToTensorV2(p=1.0),
], p=1.)
def get_valid_transforms() :
return Compose([
CenterCrop(CFG['img_size1'], CFG['img_size1'], p=1.) ,
Resize(CFG['img_size1'], CFG['img_size1']),
Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),
ToTensorV2(p=1.0),
], p=1.)
def get_inference_transforms1() :
return Compose([
OneOf([
Resize(CFG['img_size1'], CFG['img_size1'], p=1.) ,
CenterCrop(CFG['img_size1'], CFG['img_size1'], p=1.) ,
RandomResizedCrop(CFG['img_size1'], CFG['img_size1'], p=1.)
], p=1.) ,
Transpose(p=0.5),
HorizontalFlip(p=0.5),
VerticalFlip(p=0.5),
Resize(CFG['img_size1'], CFG['img_size1']),
Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),
ToTensorV2(p=1.0),
], p=1.)
def get_inference_transforms2() :
return Compose([
OneOf([
Resize(CFG['img_size2'], CFG['img_size2'], p=1.) ,
CenterCrop(CFG['img_size2'], CFG['img_size2'], p=1.) ,
RandomResizedCrop(CFG['img_size2'], CFG['img_size2'], p=1.)
], p=1.) ,
Transpose(p=0.5),
HorizontalFlip(p=0.5),
VerticalFlip(p=0.5),
Resize(CFG['img_size2'], CFG['img_size2']),
Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),
ToTensorV2(p=1.0),
], p=1.)<choose_model_class>
train_data = train_df.values
test_data = test_df.values
class CassvaImgClassifier(nn.Module):
def __init__(self, model_arch, n_class, pretrained=False):
super().__init__()
self.model = timm.create_model(model_arch, pretrained=pretrained)
 if model_arch == 'regnety_040':
self.model.head = nn.Sequential(
 nn.AdaptiveAvgPool2d(( 1,1)) ,
nn.Flatten() ,
nn.Linear(1088, n_class)
)
 elif model_arch == 'regnety_320':
self.model.head = nn.Sequential(
 nn.AdaptiveAvgPool2d(( 1,1)) ,
nn.Flatten() ,
nn.Linear(3712, n_class)
)
 elif model_arch == 'regnety_080':
self.model.head = nn.Sequential(
 nn.AdaptiveAvgPool2d(( 1,1)) ,
nn.Flatten() ,
nn.Linear(2016, n_class)
)
 elif model_arch == 'regnety_160':
self.model.head = nn.Sequential(
 nn.AdaptiveAvgPool2d(( 1,1)) ,
nn.Flatten() ,
nn.Linear(3024, n_class)
)
 else:
n_features = self.model.classifier.in_features
self.model.classifier = nn.Linear(n_features, n_class)
 def forward(self, x):
x = self.model(x)
 return x<choose_model_class>
labels = train_data[:,0]
train = train_data[:,1:]/255
class CassvaImgClassifier_ViT(nn.Module):
def __init__(self, model_arch, n_class, pretrained=False):
super().__init__()
self.model = timm.create_model(model_arch, pretrained=pretrained)
 self.model.head = nn.Linear(self.model.head.in_features, n_class)
 for module in self.model.modules() :
if isinstance(module, nn.BatchNorm2d):
if hasattr(module, 'weight'):
module.weight.requires_grad_(False)
 if hasattr(module, 'bias'):
module.bias.requires_grad_(False)
 def forward(self, x):
x = self.model(x)
 return x<create_dataframe>
dummy_y = keras.utils.to_categorical(labels)
 x_train, x_test, y_train, y_test = train_test_split(train, dummy_y, test_size=0.1, random_state=166,stratify=labels )
def prepare_dataloader(df, trn_idx, val_idx, data_root='.. /input/cassava-leaf-disease-classification/train_images/'):
train_ = df.loc[trn_idx,:].reset_index(drop=True)
 valid_ = df.loc[val_idx,:].reset_index(drop=True)
 train_ds = CassavaDataset(train_, data_root, transforms=get_train_transforms() , output_label=True)
 valid_ds = CassavaDataset(valid_, data_root, transforms=get_valid_transforms() , output_label=True)
 train_loader = torch.utils.data.DataLoader(
 train_ds,
batch_size=CFG['train_bs'],
pin_memory=False,
drop_last=False,
shuffle=True,
num_workers=CFG['num_workers'], 
)
 val_loader = torch.utils.data.DataLoader(
 valid_ds,
batch_size=CFG['valid_bs'],
num_workers=CFG['num_workers'],
shuffle=False,
pin_memory=False, 
)
 return train_loader, val_loader
def train_one_epoch(epoch, model, loss_fn, optimizer, train_loader, device, scheduler=None, schd_batch_update=False):
model.train()
t = time.time()
running_loss = None
for step,(imgs, image_labels)in enumerate(train_loader):
imgs = imgs.to(device ).float()
image_labels = image_labels.to(device ).long()
with autocast() :
image_preds = model(imgs)
 loss = loss_fn(image_preds, image_labels)
 scaler.scale(loss ).backward()
if running_loss is None:
running_loss = loss.item()
else:
running_loss = running_loss *.99 + loss.item() *.01
if(( step + 1)% CFG['accum_iter'] == 0)or(( step + 1)== len(train_loader)) :
scaler.step(optimizer)
 scaler.update()
optimizer.zero_grad()
if scheduler is not None and schd_batch_update:
scheduler.step()
if scheduler is not None and not schd_batch_update:
scheduler.step()
def valid_one_epoch(epoch, model, loss_fn, val_loader, device, scheduler=None, schd_loss_update=False):
model.eval()
t = time.time()
loss_sum = 0
sample_num = 0
image_preds_all = []
image_targets_all = []
for step,(imgs, image_labels)in enumerate(val_loader):
imgs = imgs.to(device ).float()
image_labels = image_labels.to(device ).long()
image_preds = model(imgs)
 image_preds_all += [torch.argmax(image_preds, 1 ).detach().cpu().numpy() ]
image_targets_all += [image_labels.detach().cpu().numpy() ]
loss = loss_fn(image_preds, image_labels)
 loss_sum += loss.item() *image_labels.shape[0]
sample_num += image_labels.shape[0]
image_preds_all = np.concatenate(image_preds_all)
 image_targets_all = np.concatenate(image_targets_all)
 print('epoch = {}'.format(epoch+1), 'validation multi-class accuracy = {:.4f}'.format(( image_preds_all==image_targets_all ).mean()))
if scheduler is not None:
if schd_loss_update:
scheduler.step(loss_sum/sample_num)
 else:
scheduler.step()
def inference_one_epoch(model, data_loader, device):
model.eval()
image_preds_all = []
with torch.no_grad() :
for step,(imgs)in enumerate(data_loader):
imgs = imgs.to(device ).float()
image_preds = model(imgs)
 image_preds_all += [torch.softmax(image_preds, 1 ).detach().cpu().numpy() ]
image_preds_all = np.concatenate(image_preds_all, axis=0)
 return image_preds_all<train_model>
model = Sequential()
callbacks = [keras.callbacks.ModelCheckpoint('minist.h5', monitor='val_acc', verbose=1, save_best_only=True,
mode='auto')]
model.add(Conv2D(64, kernel_size=(3, 3),
activation='relu',padding='same',
input_shape=(28,28,1)))
 model.add(Conv2D(64,(3, 3),padding='same', activation='relu'))
model.add(Conv2D(128,(3, 3),padding='same', activation='relu'))
model.add(Conv2D(128,(28, 28),activation='relu'))
model.add(Flatten())
 model.add(Dense(256, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(10, activation='softmax'))
model.summary()
sgd = SGD(lr=0.01, momentum=0.9)
 model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy'])
 model.fit(x_train, y_train,
batch_size=128,
epochs=50,
verbose=1,
validation_data=(x_test, y_test),callbacks=callbacks )
def freeze_batchnorm_stats(net):
try:
for m in net.modules() :
if isinstance(m,nn.BatchNorm2d)or isinstance(m,nn.LayerNorm):
m.eval()
except ValuError:
print('error with batchnorm2d or layernorm')
 return
def unfreeze_batchnorm_stats(net):
try:
for m in net.modules() :
if isinstance(m,nn.BatchNorm2d)or isinstance(m,nn.LayerNorm):
m.train()
except ValuError:
print('error with batchnorm2d or layernorm')
 return<normalization>
model.load_weights('minist.h5' )
class LabelSmoothingCrossEntropy(nn.Module):

 def __init__(self, smoothing=0.1):

 super(LabelSmoothingCrossEntropy, self ).__init__()
assert smoothing < 1.0
self.smoothing = smoothing
self.confidence = 1.- smoothing
def forward(self, x, target):
logprobs = torch.nn.functional.log_softmax(x, dim=-1)
 nll_loss = -logprobs.gather(dim=-1, index=target.unsqueeze(1))
nll_loss = nll_loss.squeeze(1)
 smooth_loss = -logprobs.mean(dim=-1)
 loss = self.confidence * nll_loss + self.smoothing * smooth_loss
return loss.mean()<feature_engineering>
predict = model.predict(test )
if __name__ == '__main__':
seed_everything(CFG['seed'])
 oof_preds = np.zeros(len(train))
print('Model 1 Start')
 sub1 = []
folds = StratifiedKFold(n_splits=CFG['fold_num'] ).split(np.arange(train.shape[0]), train.label.values)
 for fold,(trn_idx, val_idx)in enumerate(folds):
print('Inference fold {} started'.format(fold))
test = pd.DataFrame()
test['image_id'] = list(os.listdir('.. /input/cassava-leaf-disease-classification/test_images/'))
test_ds2 = CassavaDataset(test, '.. /input/cassava-leaf-disease-classification/test_images/', transforms=get_inference_transforms2() , output_label=False)
 tst_loader2 = torch.utils.data.DataLoader(
 test_ds2,
batch_size=CFG['valid_bs'],
num_workers=CFG['num_workers'],
shuffle=False,
pin_memory=False, 
)
 device = torch.device(CFG['device'])
 model1 = CassvaImgClassifier(CFG['model_arch1'], train.label.nunique() ).to(device)
 tst_preds = []
model1.load_state_dict(torch.load('.. /input/leaf-weight-v9-2/model9_2/swa_{}_fold_{}_{}'.format(CFG['model_arch1'], fold, '9')))
 for tta in range(CFG['tta_num']):
tst_preds += [inference_one_epoch(model1, tst_loader2, device)]
sub1 += [np.mean(tst_preds, axis=0)]
del model1;
torch.cuda.empty_cache()
print('Model 2 Start')
 sub2 = []
folds = StratifiedKFold(n_splits=CFG['fold_num'] ).split(np.arange(train.shape[0]), train.label.values)
 for fold,(trn_idx, val_idx)in enumerate(folds):
print('Inference fold {} started'.format(fold))
test = pd.DataFrame()
test['image_id'] = list(os.listdir('.. /input/cassava-leaf-disease-classification/test_images/'))
test_ds2 = CassavaDataset(test, '.. /input/cassava-leaf-disease-classification/test_images/', transforms=get_inference_transforms2() , output_label=False)
 tst_loader2 = torch.utils.data.DataLoader(
 test_ds2,
batch_size=CFG['valid_bs'],
num_workers=CFG['num_workers'],
shuffle=False,
pin_memory=False, 
)
 device = torch.device(CFG['device'])
 model2 = CassvaImgClassifier(CFG['model_arch2'], train.label.nunique() ).to(device)
 tst_preds = []
model2.load_state_dict(torch.load('.. /input/905-training-efficientnetb4-merged-bs32/swa_{}_fold_{}_{}'.format(CFG['model_arch2'], fold, '9')))
 for tta in range(CFG['tta_num']):
tst_preds += [inference_one_epoch(model2, tst_loader2, device)]
sub2 += [np.mean(tst_preds, axis=0)]
del model2;
torch.cuda.empty_cache()
print('Model 3 Start')
 sub3 = []
folds = StratifiedKFold(n_splits=CFG['fold_num'] ).split(np.arange(train.shape[0]), train.label.values)
 for fold,(trn_idx, val_idx)in enumerate(folds):
print('Inference fold {} started'.format(fold))
test = pd.DataFrame()
test['image_id'] = list(os.listdir('.. /input/cassava-leaf-disease-classification/test_images/'))
test_ds2 = CassavaDataset(test, '.. /input/cassava-leaf-disease-classification/test_images/', transforms=get_inference_transforms2() , output_label=False)
 tst_loader2 = torch.utils.data.DataLoader(
 test_ds2,
batch_size=CFG['valid_bs'],
num_workers=CFG['num_workers'],
shuffle=False,
pin_memory=False, 
)
 device = torch.device(CFG['device'])
 model3 = CassvaImgClassifier(CFG['model_arch3'], train.label.nunique() ).to(device)
 tst_preds = []
model3.load_state_dict(torch.load('.. /input/regnety4noresetadamp/swa_{}_fold_{}_{}'.format(CFG['model_arch3'], fold, '19')))
 for tta in range(CFG['tta_num']):
tst_preds += [inference_one_epoch(model3, tst_loader2, device)]
sub3 += [np.mean(tst_preds, axis=0)]
del model3;
torch.cuda.empty_cache()
print('Model 4 Start')
 sub4 = []
folds = StratifiedKFold(n_splits=CFG['fold_num'] ).split(np.arange(train.shape[0]), train.label.values)
 for fold,(trn_idx, val_idx)in enumerate(folds):
print('Inference fold {} started'.format(fold))
test = pd.DataFrame()
test['image_id'] = list(os.listdir('.. /input/cassava-leaf-disease-classification/test_images/'))
test_ds2 = CassavaDataset(test, '.. /input/cassava-leaf-disease-classification/test_images/', transforms=get_inference_transforms2() , output_label=False)
 tst_loader2 = torch.utils.data.DataLoader(
 test_ds2,
batch_size=CFG['valid_bs'],
num_workers=CFG['num_workers'],
shuffle=False,
pin_memory=False, 
)
 device = torch.device(CFG['device'])
 model4 = CassvaImgClassifier(CFG['model_arch4'], train.label.nunique() ).to(device)
 tst_preds = []
model4.load_state_dict(torch.load('.. /input/0214v1-hwkim-regnet-40-reset-swalr-swastep-ep24/swa_{}_fold_{}_{}'.format(CFG['model_arch4'], fold, '23')))
 for tta in range(CFG['tta_num']):
tst_preds += [inference_one_epoch(model4, tst_loader2, device)]
sub4 += [np.mean(tst_preds, axis=0)]
del model4;
torch.cuda.empty_cache()
print('Model 5 Start')
 sub5 = []
folds = StratifiedKFold(n_splits=CFG['fold_num'] ).split(np.arange(train.shape[0]), train.label.values)
 for fold,(trn_idx, val_idx)in enumerate(folds):
print('Inference fold {} started'.format(fold))
test = pd.DataFrame()
test['image_id'] = list(os.listdir('.. /input/cassava-leaf-disease-classification/test_images/'))
test_ds2 = CassavaDataset(test, '.. /input/cassava-leaf-disease-classification/test_images/', transforms=get_inference_transforms2() , output_label=False)
 tst_loader2 = torch.utils.data.DataLoader(
 test_ds2,
batch_size=CFG['valid_bs'],
num_workers=CFG['num_workers'],
shuffle=False,
pin_memory=False, 
)
 device = torch.device(CFG['device'])
 model5 = CassvaImgClassifier(CFG['model_arch5'], train.label.nunique() ).to(device)
 tst_preds = []
model5.load_state_dict(torch.load('.. /input/905-training-efficientnetb4-seed720/swa_{}_fold_{}_{}'.format(CFG['model_arch5'], fold, '9')))
 for tta in range(CFG['tta_num']):
tst_preds += [inference_one_epoch(model5, tst_loader2, device)]
sub5 += [np.mean(tst_preds, axis=0)]
del model5;
torch.cuda.empty_cache()
print('Model 6 Start')
 sub6 = []
folds = StratifiedKFold(n_splits=CFG['fold_num'] ).split(np.arange(train.shape[0]), train.label.values)
 for fold,(trn_idx, val_idx)in enumerate(folds):
print('Inference fold {} started'.format(fold))
test = pd.DataFrame()
test['image_id'] = list(os.listdir('.. /input/cassava-leaf-disease-classification/test_images/'))
test_ds2 = CassavaDataset(test, '.. /input/cassava-leaf-disease-classification/test_images/', transforms=get_inference_transforms2() , output_label=False)
 tst_loader2 = torch.utils.data.DataLoader(
 test_ds2,
batch_size=CFG['valid_bs'],
num_workers=CFG['num_workers'],
shuffle=False,
pin_memory=False, 
)
 device = torch.device(CFG['device'])
 model6 = CassvaImgClassifier(CFG['model_arch6'], train.label.nunique() ).to(device)
 tst_preds = []
model6.load_state_dict(torch.load('.. /input/reg-distill/swa_{}_fold_{}_{}'.format(CFG['model_arch6'], fold, '19')))
 for tta in range(CFG['tta_num']):
tst_preds += [inference_one_epoch(model6, tst_loader2, device)]
sub6 += [np.mean(tst_preds, axis=0)]
del model6;
torch.cuda.empty_cache()
sub1 = [e * CFG['weight'][0] for e in sub1]
sub2 = [e * CFG['weight'][1] for e in sub2]
sub3 = [e * CFG['weight'][2] for e in sub3]
sub4 = [e * CFG['weight'][3] for e in sub4]
sub5 = [e * CFG['weight'][4] for e in sub5]
sub6 = [e * CFG['weight'][5] for e in sub6]
sub = [e1 + e2 + e3 + e4 + e5 + e6 for(e1, e2, e3, e4, e5, e6)in zip(sub1,sub2,sub3, sub4, sub5, sub6)]<feature_engineering>
results = np.argmax(predict,axis = 1 )
test['label'] = np.argmax(np.mean(sub, axis=0), axis=1)
 test.head()<save_to_csv>
submission = pd.DataFrame({"ImageId":range(1,28001),"Label":results})
 submission.to_csv("cnn_mnist.csv",index=False )