Kaggle competition: Cassava Leaf Disease Classification
<SOS> metric: CategorizationAccuracy Kaggle data source: cassava-leaf-disease-classification<data_type_conversions>
package_paths = ['.. /input/imgclsmob/imgclsmob-master',
'.. /input/efficientnetpytorch/EfficientNet-PyTorch-master',
'.. /input/timm-pytorch-image-models/pytorch-image-models-master']
for package_path in package_paths:
sys.path.append(package_path )
tgs = ".. /input"
tgs1 = ".. /input/tgs-salt-identification-challenge"
def getImage(imgId):
path = Path(tgs+"/train/images/")/ '{}'.format(imgId)
 img = imread(path)
 return img.astype(np.uint8)
 def getGrayImage(imgId):
path = Path(tgs+"/train/images/")/'{}'.format(imgId)
 img = imread(path ).astype(np.uint8)
 img = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
 return img
def getGrayTestImage(imgId):
path = Path(tgs+"/test/images/")/'{}'.format(imgId)
 img = imread(path ).astype(np.uint8)
 img = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
 return img
def getMask(imgId):
path = Path(tgs+"/train/masks/")/ '{}'.format(imgId)
 img = imread(path)
 return img.astype(np.uint8 )<statistical_test>
import numpy as np
import pandas as pd
import os
import torch
import random
from albumentations import *
from albumentations.pytorch import ToTensorV2
from torch.utils.data import Dataset, DataLoader
import timm
import torch.nn as nn
import torch.nn.functional as F
from efficientnet_pytorch import EfficientNet
from pytorch.pytorchcv.model_provider import get_model as ptcv_get_model
from tqdm import tqdm
thresholds = [0.5, 0.55, 0.60, 0.65, 0.70, 0.75, 0.80, 0.85, 0.90, 0.95]
n_thresholds = len(thresholds)
 def IoUhelper(TrueMask, predictedMask):
intersection = cv2.bitwise_and(TrueMask, predictedMask)
 union = cv2.bitwise_or(TrueMask, predictedMask)
 intersectionCnt = cv2.countNonZero(intersection)
 unionCnt = cv2.countNonZero(union)
 return intersectionCnt, unionCnt
def meanHit(TrueMask, predictedMask):
hitCnt = 0
intersectionCnt, unionCnt = IoUhelper(TrueMask, predictedMask)
 if(intersectionCnt == 0 and unionCnt == 0):
return 1
if(intersectionCnt == 0 and unionCnt != 0):
return 0
IoU = intersectionCnt/unionCnt
for t in thresholds:
hitCnt+=int(IoU > t)
 return hitCnt/n_thresholds
def accuracy(mask, mask_pred):
num = mask.shape[0]
scores = []
for i in range(num):
score = meanHit(mask[i], mask_pred[i])
 scores.append(score)
 return np.mean(scores)
 def mean_iou(y_true, y_pred):
prec = []
y_pred_ = tf.to_int32(y_pred > 0.55)
 y_true = tf.to_int32(y_true > 0.55)
 for t in np.arange(0.5, 1.0, 0.05):
score, up_opt = tf.metrics.mean_iou(y_true, y_pred_, 2)
 K.get_session().run(tf.local_variables_initializer())
 with tf.control_dependencies([up_opt]):
score = tf.identity(score)
 prec.append(score)
 return K.mean(K.stack(prec), axis=0)
 def get_iou_vector(A, B):
batch_size = A.shape[0]
metric = []
for batch in range(batch_size):
t, p = A[batch]>0, B[batch]>0
intersection = np.logical_and(t, p)
 union = np.logical_or(t, p)
 iou =(np.sum(intersection > 0)+ 1e-10)/(np.sum(union > 0)+ 1e-10)
 thresholds = np.arange(0.5, 1, 0.05)
 s = []
for thresh in thresholds:
s.append(iou > thresh)
 metric.append(np.mean(s))
return np.mean(metric)
 def my_iou_metric(label, pred):
return -tf.py_func(get_iou_vector, [label, pred>0.5], tf.float64)
 def IoU(y_true, y_pred, eps=1e-6):
if np.max(y_true)== 0.0:
return IoU(1-y_true, 1-y_pred)
 intersection = K.sum(y_true * y_pred, axis=[1,2,3])
 union = K.sum(y_true, axis=[1,2,3])+ K.sum(y_pred, axis=[1,2,3])- intersection
return -K.mean(( intersection + eps)/(union + eps), axis=0 )<categorify>
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
def rle_encoding(mask):
mask = mask.ravel()
encoding = ""
i = 0
while(i<len(mask)) :
currCnt = 0
start = i
if(mask[i] == 255):
while(i < len(mask)and mask[i] == 255):
currCnt+=1
i+=1
encoding+=(" "+ str(start+1)+ " " + str(currCnt))
else: i+=1
return encoding.strip()<categorify>
class CasDataset(Dataset):
def __init__(self, df, path, transforms):
self.df = df
self.path = path
self.transforms = transforms
def __len__(self):
return self.df.shape[0]
def __getitem__(self, idx):
image = cv2.imread(self.path+self.df.loc[idx, 'image_id'])
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
 if self.transforms:
transformed = self.transforms(image=image)
 image = transformed['image']
return image
imgId = train[haveMask].index[18]
rle_orig = train[haveMask].rle_mask[18]
img1 = getImage(imgId+".png")
 img2 = getMask(imgId+".png")
 rle1 = rle_encoding(np.transpose(img2))
print("Original Encoding
", rle_orig)
 print("Len1 = ", len(rle_orig.split()))
print("-------")
 print("Encoding
", rle1)
 print("Len2 = ", len(rle1.split()))<define_variables>
class CasModel(nn.Module):
def __init__(self, num_classes=5, model="resnet34"):
super().__init__()
if model == "resnet16":
self.backbone = ptcv_get_model("resnet16", pretrained=False)
 self.backbone.features.final_pool = nn.AdaptiveAvgPool2d(1)
 self.backbone.output = nn.Sequential(nn.Linear(512, num_classes))
elif model == "resnet34":
self.backbone = ptcv_get_model("resnet34", pretrained=False)
 self.backbone.features.final_pool = nn.AdaptiveAvgPool2d(1)
 self.backbone.output = nn.Sequential(nn.Linear(512, num_classes))
elif model == "efficientnet-b2":
self.backbone = EfficientNet.from_pretrained("efficientnet-b2")
 self.backbone._fc = nn.Linear(self.backbone._fc.in_features, num_classes)
 elif model == "efficientnet-b4":
self.backbone = EfficientNet.from_pretrained("efficientnet-b4")
 self.backbone._fc = nn.Linear(self.backbone._fc.in_features, num_classes)
 elif model == "efficientnet-b7":
self.backbone = EfficientNet.from_pretrained("efficientnet-b7")
 self.backbone._fc = nn.Linear(self.backbone._fc.in_features, num_classes)
 elif model == "vit-b":
self.backbone = timm.create_model("vit_base_patch16_384", pretrained=False)
 self.backbone.head = nn.Linear(self.backbone.head.in_features, num_classes)
 elif model == "vit-l":
self.backbone = timm.create_model("vit_large_patch16_384", pretrained=False)
 self.backbone.head = nn.Linear(self.backbone.head.in_features, num_classes)
 elif model == "vit-r":
self.backbone = timm.create_model("vit_base_resnet50d_224", pretrained=False)
 self.backbone.head = nn.Linear(self.backbone.head.in_features, num_classes)
 elif model == "efficientnet-b4-timm":
self.backbone = timm.create_model("tf_efficientnet_b4_ns", pretrained=False)
 self.backbone.classifier = nn.Linear(self.backbone.classifier.in_features, num_classes)
 else:
raise NotImplementedError
def forward(self, x):
x = self.backbone(x)
 return x
TRAIN_IMAGE_DIR = tgs+'/train/images'
TRAIN_MASK_DIR = tgs+'/train/masks'
TEST_MASK_DIR = tgs+'/test/images'
im_height = 128
im_width = 128
train_image_list = os.listdir(TRAIN_IMAGE_DIR)
 train_mask_list = os.listdir(TRAIN_MASK_DIR)
 test_image_list = os.listdir(TEST_MASK_DIR )<prepare_x_and_y>
def inference_one_epoch(model, data_loader, device):
model.eval()
image_preds_all = []
for step,(imgs)in enumerate(data_loader):
imgs = imgs.to(device ).float()
image_preds = model(imgs)
 image_preds_all += [torch.softmax(image_preds, 1 ).detach().cpu().numpy() ]
image_preds_all = np.concatenate(image_preds_all, axis=0)
 return image_preds_all
X_train_image = np.zeros(( len(train_image_list),
im_height, im_width, 1), dtype = np.uint8)
 Y_train_mask = np.zeros(( len(train_mask_list),
im_height, im_width, 1), dtype = np.uint8)
 for i in tqdm(range(len(train_image_list))):
imgId = train_image_list[i]
img = getGrayImage(imgId)
 img = cv2.resize(img,(im_height, im_width))
X_train_image[i] = img.reshape(im_height, im_width, 1)
 img = getMask(imgId)
 img = cv2.resize(img,(im_height, im_width))
Y_train_mask[i] = img.reshape(im_height, im_width, 1)
 print("X_train Shape = ", X_train_image.shape, "Y_train Shape = ", Y_train_mask.shape )<concatenate>
seed_everything(960630)
 test_df = pd.DataFrame()
test_df['image_id'] = list(os.listdir('.. /input/cassava-leaf-disease-classification/test_images/'))
test_data = CasDataset(test_df, '.. /input/cassava-leaf-disease-classification/test_images/', transforms_test )
<count_values>
test_loader = DataLoader(
 test_data,
batch_size=32,
num_workers=8,
shuffle=False,
pin_memory=False,
)
print(np.count_nonzero(Y_train_mask[2]))
print(cntOne[2] )<count_unique_values>
test_preds = []
device = torch.device('cuda:0')
 model = CasModel(model="resnet34" ).to(device)
 for fold in range(5):
model.load_state_dict(torch.load(f'.. /input/casresnet34/resnet34_f_{fold}.pth'))
with torch.no_grad() :
for t in range(tta):
test_preds += [inference_one_epoch(model, test_loader, device)]
x = np.array([0, 1.0, 3.0, 1.6])
 bins = np.array([0, 1.0, 2.5, 4.0, 10.0])
 inds = np.digitize(x, bins)
 np.unique(inds )<count_unique_values>
test_preds = np.mean(test_preds, axis=0)
 test_df['label'] = np.argmax(test_preds, axis=1)
 test_df.head()
<split><EOS> 
test_df.to_csv('submission.csv', index=False )