Kaggle competition: iMet Collection 2019 - FGVC6
<SOS> metric: MeanFScoreBeta Kaggle data source: imet-2019-fgvc6<import_modules>
fastai.__version__
import segmentation_models_pytorch as smp<categorify>
path = Path('.. /input/imet-2019-fgvc6/' )
def run_length_decode(rle, height=1024, width=1024, fill_value=1):
component = np.zeros(( height, width), np.float32)
 component = component.reshape(-1)
 rle = np.array([int(s)for s in rle.strip().split(' ')])
 rle = rle.reshape(-1, 2)
 start = 0
for index, length in rle:
start = start+index
end = start+length
component[start: end] = fill_value
start = end
component = component.reshape(width, height ).T
return component
def run_length_encode(component):
component = component.T.flatten()
start = np.where(component[1:] > component[:-1])[0]+1
end = np.where(component[:-1] > component[1:])[0]+1
length = end-start
rle = []
for i in range(len(length)) :
if i == 0:
rle.extend([start[0], length[0]])
 else:
rle.extend([start[i]-end[i-1], length[i]])
 rle = ' '.join([str(r)for r in rle])
 return rle<categorify>
Path('models' ).mkdir(exist_ok=True)
 !cp '.. /input/densenet201/densenet201.pth' 'models/'
def load_url(*args, **kwargs):
model_dir = Path('models')
 filename = 'densenet201.pth'
if not(model_dir/filename ).is_file() : raise FileNotFoundError
return torch.load(model_dir/filename)
 model_zoo.load_url = load_url
class SIIMDataset(Dataset):
def __init__(self, df, fnames, data_folder, size, mean, std, phase):
self.df = df
self.root = data_folder
self.size = size
self.mean = mean
self.std = std
self.phase = phase
self.transforms = get_transforms(phase, size, mean, std)
 self.gb = self.df.groupby('ImageId')
 self.fnames = fnames
def __getitem__(self, idx):
image_id = self.fnames[idx]
df = self.gb.get_group(image_id)
 annotations = df[' EncodedPixels'].tolist()
image_path = os.path.join(self.root, image_id + ".png")
 image = cv2.imread(image_path)
 mask = np.zeros([1024, 1024])
 if annotations[0] != ' -1':
for rle in annotations:
mask += run_length_decode(rle)
 mask =(mask >= 1 ).astype('float32')
 augmented = self.transforms(image=image, mask=mask)
 image = augmented['image']
mask = augmented['mask']
return image, mask
def __len__(self):
return len(self.fnames)
 def get_transforms(phase, size, mean, std):
list_transforms = []
if phase == "train":
list_transforms.extend(
 [
ShiftScaleRotate(
 shift_limit=0,
scale_limit=0.1,
rotate_limit=10,
p=0.5,
border_mode=cv2.BORDER_CONSTANT 
),
] 
)
 list_transforms.extend(
 [
Resize(size, size),
Normalize(mean=mean, std=std, p=1),
ToTensor() ,
] 
)
 list_trfms = Compose(list_transforms)
 return list_trfms
def provider(
 fold,
total_folds,
data_folder,
df_path,
phase,
size,
mean=None,
std=None,
batch_size=8,
num_workers=4, 
):
df_all = pd.read_csv(df_path)
 df = df_all.drop_duplicates('ImageId')
 df_with_mask = df[df[" EncodedPixels"] != " -1"]
df_with_mask['has_mask'] = 1
df_without_mask = df[df[" EncodedPixels"] == " -1"]
df_without_mask['has_mask'] = 0
df_without_mask_sampled = df_without_mask.sample(len(df_with_mask), random_state=69)
 df = pd.concat([df_with_mask, df_without_mask_sampled])
 kfold = StratifiedKFold(total_folds, shuffle=True, random_state=69)
 train_idx, val_idx = list(kfold.split(df["ImageId"], df["has_mask"])) [fold]
train_df, val_df = df.iloc[train_idx], df.iloc[val_idx]
df = train_df if phase == "train" else val_df
fnames = df['ImageId'].values
image_dataset = SIIMDataset(df_all, fnames, data_folder, size, mean, std, phase)
 dataloader = DataLoader(
 image_dataset,
batch_size=batch_size,
num_workers=num_workers,
pin_memory=True,
shuffle=True, 
)
 return dataloader<define_variables>
train_df = pd.read_csv(path/'train.csv')
 train_df.head()
sample_submission_path = '.. /input/siim-stage1/sample_submission.csv'
train_rle_path = '.. /input/siim-dicom-images/train-rle.csv'
data_folder = ".. /input/siim-png-images/input/train_png"
test_data_folder = ".. /input/siim-png-images/input/test_png"<define_variables>
labels_df = pd.read_csv(path/'labels.csv')
 labels_df.head()
dataloader = provider(
 fold=0,
total_folds=5,
data_folder=data_folder,
df_path=train_rle_path,
phase="train",
size=512,
mean =(0.485, 0.456, 0.406),
std =(0.229, 0.224, 0.225),
batch_size=16,
num_workers=4,
)<define_variables>
test_df = pd.read_csv(path/'sample_submission.csv')
 test_df.head()
batch = next(iter(dataloader))
images, masks = batch<compute_test_metric>
train, test = [ImageList.from_df(df, path=path, cols='id', folder=folder, suffix='.png')
 for df, folder in zip([train_df, test_df], ['train', 'test'])]
data =(train.split_by_rand_pct(0.2, seed=42)
.label_from_df(cols='attribute_ids', label_delim=' ')
.add_test(test)
.transform(get_transforms() , size=128)
.databunch(path=Path('.'), bs=64 ).normalize() )
def dice_loss(input, target):
input = torch.sigmoid(input)
 smooth = 1.0
iflat = input.view(-1)
 tflat = target.view(-1)
 intersection =(iflat * tflat ).sum()
return(( 2.0 * intersection + smooth)/(iflat.sum() + tflat.sum() + smooth))
class FocalLoss(nn.Module):
def __init__(self, gamma):
super().__init__()
self.gamma = gamma
def forward(self, input, target):
if not(target.size() == input.size()):
raise ValueError("Target size({})must be the same as input size({})" 
.format(target.size() , input.size()))
max_val =(-input ).clamp(min=0)
 loss = input - input * target + max_val + \ 
(( -max_val ).exp() +(-input - max_val ).exp() ).log()
invprobs = F.logsigmoid(-input *(target * 2.0 - 1.0))
loss =(invprobs * self.gamma ).exp() * loss
return loss.mean()
class MixedLoss(nn.Module):
def __init__(self, alpha, gamma):
super().__init__()
self.alpha = alpha
self.focal = FocalLoss(gamma)
 def forward(self, input, target):
loss = self.alpha*self.focal(input, target)- torch.log(dice_loss(input, target))
return loss.mean()<predict_on_test>
data.show_batch()
def predict(X, threshold):
X_p = np.copy(X)
 preds =(X_p > threshold ).astype('uint8')
 return preds
def metric(probability, truth, threshold=0.5, reduction='none'):


batch_size = len(truth)
 with torch.no_grad() :
probability = probability.view(batch_size, -1)
 truth = truth.view(batch_size, -1)
 assert(probability.shape == truth.shape)
 p =(probability > threshold ).float()
t =(truth > 0.5 ).float()
t_sum = t.sum(-1)
 p_sum = p.sum(-1)
 neg_index = torch.nonzero(t_sum == 0)
 pos_index = torch.nonzero(t_sum >= 1)
 dice_neg =(p_sum == 0 ).float()
dice_pos = 2 *(p*t ).sum(-1)/(( p+t ).sum(-1))
dice_neg = dice_neg[neg_index]
dice_pos = dice_pos[pos_index]
dice = torch.cat([dice_pos, dice_neg])
 num_neg = len(neg_index)
 num_pos = len(pos_index)
 return dice, dice_neg, dice_pos, num_neg, num_pos
class Meter:

 def __init__(self, phase, epoch):
self.base_threshold = 0.5
self.base_dice_scores = []
self.dice_neg_scores = []
self.dice_pos_scores = []
self.iou_scores = []
def update(self, targets, outputs):
probs = torch.sigmoid(outputs)
 dice, dice_neg, dice_pos, _, _ = metric(probs, targets, self.base_threshold)
 self.base_dice_scores.extend(dice)
 self.dice_pos_scores.extend(dice_pos)
 self.dice_neg_scores.extend(dice_neg)
 preds = predict(probs, self.base_threshold)
 iou = compute_iou_batch(preds, targets, classes=[1])
 self.iou_scores.append(iou)
 def get_metrics(self):
dice = np.nanmean(self.base_dice_scores)
 dice_neg = np.nanmean(self.dice_neg_scores)
 dice_pos = np.nanmean(self.dice_pos_scores)
 dices = [dice, dice_neg, dice_pos]
iou = np.nanmean(self.iou_scores)
 return dices, iou
def epoch_log(phase, epoch, epoch_loss, meter, start):

 dices, iou = meter.get_metrics()
dice, dice_neg, dice_pos = dices
print("Loss: %0.4f | dice: %0.4f | dice_neg: %0.4f | dice_pos: %0.4f | IoU: %0.4f" %(epoch_loss, dice, dice_neg, dice_pos, iou))
return dice, iou
def compute_ious(pred, label, classes, ignore_index=255, only_present=True):

 pred[label == ignore_index] = 0
ious = []
for c in classes:
label_c = label == c
if only_present and np.sum(label_c)== 0:
ious.append(np.nan)
 continue
pred_c = pred == c
intersection = np.logical_and(pred_c, label_c ).sum()
union = np.logical_or(pred_c, label_c ).sum()
if union != 0:
ious.append(intersection / union)
 return ious if ious else [1]
def compute_iou_batch(outputs, labels, classes=None):

 ious = []
preds = np.copy(outputs)
 labels = np.array(labels)
 for pred, label in zip(preds, labels):
ious.append(np.nanmean(compute_ious(pred, label, classes)))
 iou = np.nanmean(ious)
 return iou 
<load_pretrained>
class FocalLoss(nn.Module):
def __init__(self, gamma=2):
super().__init__()
self.gamma = gamma
def forward(self, input, target):
if not(target.size() == input.size()):
raise ValueError("Target size({})must be the same as input size({})" 
.format(target.size() , input.size()))
max_val =(-input ).clamp(min=0)
 loss = input - input * target + max_val + \ 
(( -max_val ).exp() +(-input - max_val ).exp() ).log()
invprobs = F.logsigmoid(-input *(target * 2.0 - 1.0))
loss =(invprobs * self.gamma ).exp() * loss
return loss.sum(dim=1 ).mean()
class FbetaLoss(nn.Module):
def __init__(self, beta=1):
super(FbetaLoss, self ).__init__()
self.small_value = 1e-6
self.beta = beta
def forward(self, logits, labels):
beta = self.beta
batch_size = logits.size() [0]
p = F.sigmoid(logits)
 l = labels
num_pos = torch.sum(p, 1)+ self.small_value
num_pos_hat = torch.sum(l, 1)+ self.small_value
tp = torch.sum(l * p, 1)
 precise = tp / num_pos
recall = tp / num_pos_hat
fs =(1 + beta * beta)* precise * recall /(beta * beta * precise + recall + self.small_value)
 loss = fs.sum() / batch_size
return 1 - loss
class CombineLoss(nn.Module):
def __init__(self):
super(CombineLoss, self ).__init__()
self.fbeta_loss = FbetaLoss(beta=2)
 self.focal_loss = FocalLoss()
def forward(self, logits, labels):
loss_beta = self.fbeta_loss(logits, labels)
 loss_focal = self.focal_loss(logits, labels)
 return 0.5 * loss_beta + 0.5 * loss_focal
model = smp.Unet("resnet34", encoder_weights="imagenet", activation=None )<choose_model_class>
learn = cnn_learner(data, base_arch=models.densenet201, loss_func=CombineLoss() , metrics=fbeta )
class Trainer(object):

 def __init__(self, model):
self.fold = 1
self.total_folds = 5
self.num_workers = 6
self.batch_size = {"train": 4, "val": 4}
self.accumulation_steps = 32 // self.batch_size['train']
self.lr = 5e-4
self.num_epochs = 40
self.best_loss = float("inf")
 self.phases = ["train", "val"]
self.device = torch.device("cuda:0")
 torch.set_default_tensor_type("torch.cuda.FloatTensor")
 self.net = model
self.criterion = MixedLoss(10.0, 2.0)
 self.optimizer = optim.Adam(self.net.parameters() , lr=self.lr)
 self.scheduler = ReduceLROnPlateau(self.optimizer, mode="min", patience=3, verbose=True)
 self.net = self.net.to(self.device)
 cudnn.benchmark = True
self.dataloaders = {
phase: provider(
 fold=1,
total_folds=5,
data_folder=data_folder,
df_path=train_rle_path,
phase=phase,
size=512,
mean=(0.485, 0.456, 0.406),
std=(0.229, 0.224, 0.225),
batch_size=self.batch_size[phase],
num_workers=self.num_workers, 
)
 for phase in self.phases
}
self.losses = {phase: [] for phase in self.phases}
self.iou_scores = {phase: [] for phase in self.phases}
self.dice_scores = {phase: [] for phase in self.phases}
def forward(self, images, targets):
images = images.to(self.device)
 masks = targets.to(self.device)
 outputs = self.net(images)
 loss = self.criterion(outputs, masks)
 return loss, outputs
def iterate(self, epoch, phase):
meter = Meter(phase, epoch)
 start = time.strftime("%H:%M:%S")
 print(f"Starting epoch: {epoch} | phase: {phase} | ⏰: {start}")
 batch_size = self.batch_size[phase]
self.net.train(phase == "train")
 dataloader = self.dataloaders[phase]
running_loss = 0.0
total_batches = len(dataloader)
 self.optimizer.zero_grad()
for itr, batch in enumerate(dataloader):
images, targets = batch
loss, outputs = self.forward(images, targets)
 loss = loss / self.accumulation_steps
if phase == "train":
loss.backward()
if(itr + 1)% self.accumulation_steps == 0:
self.optimizer.step()
self.optimizer.zero_grad()
running_loss += loss.item()
outputs = outputs.detach().cpu()
meter.update(targets, outputs)
 epoch_loss =(running_loss * self.accumulation_steps)/ total_batches
dice, iou = epoch_log(phase, epoch, epoch_loss, meter, start)
 self.losses[phase].append(epoch_loss)
 self.dice_scores[phase].append(dice)
 self.iou_scores[phase].append(iou)
 torch.cuda.empty_cache()
return epoch_loss
def start(self):
for epoch in range(self.num_epochs):
self.iterate(epoch, "train")
 state = {
"epoch": epoch,
"best_loss": self.best_loss,
"state_dict": self.net.state_dict() ,
"optimizer": self.optimizer.state_dict() ,
}
val_loss = self.iterate(epoch, "val")
 self.scheduler.step(val_loss)
 if val_loss < self.best_loss:
print("******** New optimal found, saving state ********")
 state["best_loss"] = self.best_loss = val_loss
torch.save(state, "./model.pth")
 print()<choose_model_class>
learn.lr_find()
learn.recorder.plot()
model_trainer = Trainer(model)
 model_trainer.start()<categorify>
lr = 3e-2
learn.fit_one_cycle(3, slice(lr))
class TestDataset(Dataset):
def __init__(self, root, df, size, mean, std, tta=4):
self.root = root
self.size = size
self.fnames = list(df["ImageId"])
 self.num_samples = len(self.fnames)
 self.transform = Compose(
 [
Normalize(mean=mean, std=std, p=1),
Resize(size, size),
ToTensor() ,
] 
)
 def __getitem__(self, idx):
fname = self.fnames[idx]
path = os.path.join(self.root, fname + ".png")
 image = cv2.imread(path)
 images = self.transform(image=image)["image"]
return images
def __len__(self):
return self.num_samples
def post_process(probability, threshold, min_size):
mask = cv2.threshold(probability, threshold, 1, cv2.THRESH_BINARY)[1]
num_component, component = cv2.connectedComponents(mask.astype(np.uint8))
predictions = np.zeros(( 1024, 1024), np.float32)
 num = 0
for c in range(1, num_component):
p =(component == c)
 if p.sum() > min_size:
predictions[p] = 1
num += 1
return predictions, num<load_pretrained>
lr = 1e-3
learn.fit_one_cycle(21, slice(lr/10, lr))
size = 512
mean =(0.485, 0.456, 0.406)
 std =(0.229, 0.224, 0.225)
 num_workers = 8
batch_size = 16
best_threshold = 0.5
min_size = 3500
device = torch.device("cuda:0")
 df = pd.read_csv(sample_submission_path)
 testset = DataLoader(
 TestDataset(test_data_folder, df, size, mean, std),
batch_size=batch_size,
shuffle=False,
num_workers=num_workers,
pin_memory=True, 
)
 model = model_trainer.net
model.eval()
state = torch.load('./model.pth', map_location=lambda storage, loc: storage)
 model.load_state_dict(state["state_dict"])
 encoded_pixels = []
for i, batch in enumerate(tqdm(testset)) :
preds = torch.sigmoid(model(batch.to(device)))
 preds = preds.detach().cpu().numpy() [:, 0, :, :]
for probability in preds:
if probability.shape !=(1024, 1024):
probability = cv2.resize(probability, dsize=(1024, 1024), interpolation=cv2.INTER_LINEAR)
 predict, num_predict = post_process(probability, best_threshold, min_size)
 if num_predict == 0:
encoded_pixels.append('-1')
 else:
r = run_length_encode(predict)
 encoded_pixels.append(r)
 df['EncodedPixels'] = encoded_pixels
df.to_csv('submission.csv', columns=['ImageId', 'EncodedPixels'], index=False )<set_options>
valid_preds = learn.get_preds(DatasetType.Valid)
 best_thr = find_best_fixed_threshold(*valid_preds )
!pip install albumentations > /dev/null
!pip install -U efficientnet==0.0.4
plt.style.use('seaborn-white')
 sns.set_style("white")
 seed = 10
np.random.seed(seed)
 random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.set_random_seed(seed)
 %matplotlib inline<load_pretrained>
test_preds = learn.get_preds(DatasetType.Test)
 test_df.attribute_ids = join_preds(test_preds, best_thr)
 test_df.head()
!mkdir masks
!unzip -q.. /input/data-repack-and-image-statistics/masks.zip -d masks
!mkdir train
!unzip -q.. /input/data-repack-and-image-statistics/train.zip -d train
!mkdir test
!unzip -q.. /input/data-repack-and-image-statistics/test.zip -d test<feature_engineering>
test_df.to_csv('submission.csv', index=False )