Kaggle competition: iMet Collection 2019 - FGVC6
<SOS> metric: MeanFScoreBeta Kaggle data source: imet-2019-fgvc6<define_variables>
fastai.__version__
max_images = 64
grid_width = 16
grid_height = int(max_images / grid_width)
 fig, axs = plt.subplots(grid_height, grid_width, figsize=(grid_width, grid_height))
validation_generator = DataGenerator(train_im_path = valid_im_path ,
train_mask_path=valid_mask_path,augmentations=AUGMENTATIONS_TEST,
img_size=img_size,batch_size=64,shuffle=False)
 images,masks = validation_generator.__getitem__(0)
 for i,(im, mask)in enumerate(zip(images,masks)) :
pred = preds_valid[i]
ax = axs[int(i / grid_width), i % grid_width]
ax.imshow(im[...,0], cmap="bone")
 ax.imshow(mask.squeeze() , alpha=0.5, cmap="Reds")
 ax.imshow(np.array(np.round(pred > threshold_best), dtype=np.float32), alpha=0.5, cmap="Greens")
 ax.axis('off')
 plt.suptitle("Green:Prediction , Red: Pneumothorax." )<predict_on_test>
path = Path('.. /input/imet-2019-fgvc6/' )
test_fn = glob.glob('./test/*')
 x_test = [cv2.resize(np.array(Image.open(fn)) ,(img_size,img_size)) for fn in test_fn]
x_test = np.array(x_test)
 x_test = np.array([np.repeat(im[...,None],3,2)for im in x_test])
 print(x_test.shape)
 preds_test_orig = model.predict(x_test,batch_size=batch_size)
 x_test = np.array([np.fliplr(x)for x in x_test])
 preds_test_flipped = model.predict(x_test,batch_size=batch_size)
 preds_test_flipped = np.array([np.fliplr(x)for x in preds_test_flipped])
 preds_test = 0.5*preds_test_orig + 0.5*preds_test_flipped 
<categorify>
Path('models' ).mkdir(exist_ok=True)
 !cp '.. /input/densenet201/densenet201.pth' 'models/'
def load_url(*args, **kwargs):
model_dir = Path('models')
 filename = 'densenet201.pth'
if not(model_dir/filename ).is_file() : raise FileNotFoundError
return torch.load(model_dir/filename)
 model_zoo.load_url = load_url
sys.path.insert(0, '.. /input/siim-acr-pneumothorax-segmentation')
 rles = []
i,max_img = 1,10
plt.figure(figsize=(16,4))
for p in tqdm_notebook(preds_test):
p = p.squeeze()
im = cv2.resize(p,(1024,1024))
im = im > threshold_best
if im.sum() <1024*2:
im[:] = 0
im =(im.T*255 ).astype(np.uint8)
 rles.append(mask2rle(im, 1024, 1024))
i += 1
if i<max_img:
plt.subplot(1,max_img,i)
 plt.imshow(im)
 plt.axis('off' )<prepare_output>
train_df = pd.read_csv(path/'train.csv')
 train_df.head()
ids = [o.split('/')[-1][:-4] for o in test_fn]
sub_df = pd.DataFrame({'ImageId': ids, 'EncodedPixels': rles})
 sub_df.loc[sub_df.EncodedPixels=='', 'EncodedPixels'] = '-1'
sub_df.head()<save_to_csv>
labels_df = pd.read_csv(path/'labels.csv')
 labels_df.head()
sub_df.to_csv('submission.csv', index=False )<set_options>
test_df = pd.read_csv(path/'sample_submission.csv')
 test_df.head()
!git clone https://github.com/qubvel/efficientnet.git
plt.style.use('seaborn-white')
 sns.set_style("white")
 seed = 10
np.random.seed(seed)
 random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.set_random_seed(seed)
 %matplotlib inline<load_pretrained>
train, test = [ImageList.from_df(df, path=path, cols='id', folder=folder, suffix='.png')
 for df, folder in zip([train_df, test_df], ['train', 'test'])]
data =(train.split_by_rand_pct(0.2, seed=42)
.label_from_df(cols='attribute_ids', label_delim=' ')
.add_test(test)
.transform(get_transforms() , size=128)
.databunch(path=Path('.'), bs=64 ).normalize() )
!mkdir masks
!unzip -q.. /input/data-repack-and-image-statistics/masks.zip -d masks
!mkdir train
!unzip -q.. /input/data-repack-and-image-statistics/train.zip -d train
!mkdir test
!unzip -q.. /input/data-repack-and-image-statistics/test.zip -d test<feature_engineering>
data.show_batch()
all_mask_fn = glob.glob('./masks/*')
 mask_df = pd.DataFrame()
mask_df['file_names'] = all_mask_fn
mask_df['mask_percentage'] = 0
mask_df.set_index('file_names',inplace=True)
 for fn in all_mask_fn:
mask_df.loc[fn,'mask_percentage'] = np.array(Image.open(fn)).sum() /(256*256*255)
 mask_df.reset_index(inplace=True)
 sns.distplot(mask_df.mask_percentage)
 mask_df['labels'] = 0
mask_df.loc[mask_df.mask_percentage>0,'labels'] = 1<split>
class FocalLoss(nn.Module):
def __init__(self, gamma=2):
super().__init__()
self.gamma = gamma
def forward(self, logit, target):
target = target.float()
max_val =(-logit ).clamp(min=0)
 loss = logit - logit * target + max_val + \ 
(( -max_val ).exp() +(-logit - max_val ).exp() ).log()
invprobs = F.logsigmoid(-logit *(target * 2.0 - 1.0))
loss =(invprobs * self.gamma ).exp() * loss
if len(loss.size())==2:
loss = loss.sum(dim=1)
 return loss.mean()
all_train_fn = glob.glob('./train/*')
 total_samples = len(all_train_fn)
 idx = np.arange(total_samples)
 train_fn,val_fn = train_test_split(all_train_fn,stratify=mask_df.labels,test_size=0.1,random_state=10)
 print('No.of train files:', len(train_fn))
print('No.of val files:', len(val_fn))
masks_train_fn = [fn.replace('./train','./masks')for fn in train_fn]
masks_val_fn = [fn.replace('./train','./masks')for fn in val_fn]<define_variables>
learn = cnn_learner(data, base_arch=models.densenet201, loss_func=FocalLoss() , metrics=fbeta )
train_im_path,train_mask_path = './keras_im_train','./keras_mask_train'
h,w,batch_size = 256,256,16
val_im_path,val_mask_path = './keras_im_val','./keras_mask_val'
class DataGenerator(keras.utils.Sequence):
'Generates data for Keras'
def __init__(self, train_im_path=train_im_path,train_mask_path=train_mask_path,
augmentations=None, batch_size=batch_size,img_size=256, n_channels=3, shuffle=True):
'Initialization'
self.batch_size = batch_size
self.train_im_paths = glob.glob(train_im_path+'/*')
 self.train_im_path = train_im_path
self.train_mask_path = train_mask_path
self.img_size = img_size
self.n_channels = n_channels
self.shuffle = shuffle
self.augment = augmentations
self.on_epoch_end()
def __len__(self):
'Denotes the number of batches per epoch'
return int(np.ceil(len(self.train_im_paths)/ self.batch_size))
def __getitem__(self, index):
'Generate one batch of data'
indexes = self.indexes[index*self.batch_size:min(( index+1)*self.batch_size,len(self.train_im_paths)) ]
list_IDs_im = [self.train_im_paths[k] for k in indexes]
X, y = self.data_generation(list_IDs_im)
 if self.augment is None:
return X,np.array(y)/255
else:
im,mask = [],[]
for x,y in zip(X,y):
augmented = self.augment(image=x, mask=y)
 im.append(augmented['image'])
 mask.append(augmented['mask'])
 return np.array(im),np.array(mask)/255
def on_epoch_end(self):
'Updates indexes after each epoch'
self.indexes = np.arange(len(self.train_im_paths))
if self.shuffle == True:
np.random.shuffle(self.indexes)
 def data_generation(self, list_IDs_im):
'Generates data containing batch_size samples'
X = np.empty(( len(list_IDs_im),self.img_size,self.img_size, self.n_channels))
y = np.empty(( len(list_IDs_im),self.img_size,self.img_size, 1))
for i, im_path in enumerate(list_IDs_im):
im = np.array(Image.open(im_path))
mask_path = im_path.replace(self.train_im_path,self.train_mask_path)
 mask = np.array(Image.open(mask_path))
if len(im.shape)==2:
im = np.repeat(im[...,None],3,2)
 X[i,] = cv2.resize(im,(self.img_size,self.img_size))
y[i,] = cv2.resize(mask,(self.img_size,self.img_size)) [..., np.newaxis]
y[y>0] = 255
return np.uint8(X),np.uint8(y )<normalization>
learn.lr_find()
learn.recorder.plot()
Compose, HorizontalFlip, CLAHE, HueSaturationValue,
RandomBrightness, RandomContrast, RandomGamma,OneOf,
ToFloat, ShiftScaleRotate,GridDistortion, ElasticTransform, JpegCompression, HueSaturationValue,
RGBShift, RandomBrightness, RandomContrast, Blur, MotionBlur, MedianBlur, GaussNoise,CenterCrop,
IAAAdditiveGaussianNoise,GaussNoise,OpticalDistortion,RandomSizedCrop 
)
 AUGMENTATIONS_TRAIN = Compose([
HorizontalFlip(p=0.5),
OneOf([
RandomContrast() ,
RandomGamma() ,
RandomBrightness() ,
], p=0.3),
OneOf([
ElasticTransform(alpha=120, sigma=120 * 0.05, alpha_affine=120 * 0.03),
GridDistortion() ,
OpticalDistortion(distort_limit=2, shift_limit=0.5),
], p=0.3),
RandomSizedCrop(min_max_height=(160, 256), height=h, width=w,p=0.25),
ToFloat(max_value=1)
 ],p=1)
 AUGMENTATIONS_TEST = Compose([
ToFloat(max_value=1)
 ],p=1 )<compute_train_metric>
lr = 3e-2
learn.fit_one_cycle(3, slice(lr))
def get_iou_vector(A, B):
batch_size = A.shape[0]
metric = 0.0
for batch in range(batch_size):
t, p = A[batch], B[batch]
true = np.sum(t)
 pred = np.sum(p)
 if true == 0:
metric +=(pred == 0)
 continue
intersection = np.sum(t * p)
 union = true + pred - intersection
iou = intersection / union
iou = np.floor(max(0,(iou - 0.45)*20)) / 10
metric += iou
metric /= batch_size
return metric
def my_iou_metric(label, pred):
return tf.py_func(get_iou_vector, [label, pred > 0.5], tf.float64 )<compute_test_metric>
lr = 1e-3
learn.fit_one_cycle(21, slice(lr/10, lr))
def dice_coef(y_true, y_pred):
y_true_f = K.flatten(y_true)
 y_pred = K.cast(y_pred, 'float32')
 y_pred_f = K.cast(K.greater(K.flatten(y_pred), 0.5), 'float32')
 intersection = y_true_f * y_pred_f
score = 2.* K.sum(intersection)/(K.sum(y_true_f)+ K.sum(y_pred_f))
return score
def dice_loss(y_true, y_pred):
smooth = 1.
 y_true_f = K.flatten(y_true)
 y_pred_f = K.flatten(y_pred)
 intersection = y_true_f * y_pred_f
score =(2.* K.sum(intersection)+ smooth)/(K.sum(y_true_f)+ K.sum(y_pred_f)+ smooth)
 return 1.- score
def bce_dice_loss(y_true, y_pred):
return binary_crossentropy(y_true, y_pred)+ dice_loss(y_true, y_pred)
 def bce_logdice_loss(y_true, y_pred):
return binary_crossentropy(y_true, y_pred)- K.log(1.- dice_loss(y_true, y_pred))<choose_model_class>
valid_preds = learn.get_preds(DatasetType.Valid)
 best_thr = find_best_fixed_threshold(*valid_preds )
class SnapshotCallbackBuilder:
def __init__(self, nb_epochs, nb_snapshots, init_lr=0.1):
self.T = nb_epochs
self.M = nb_snapshots
self.alpha_zero = init_lr
def get_callbacks(self, model_prefix='Model'):
callback_list = [
callbacks.ModelCheckpoint("./keras.model",monitor='val_my_iou_metric',
mode = 'max', save_best_only=True, verbose=1),
swa,
callbacks.LearningRateScheduler(schedule=self._cosine_anneal_schedule)
 ]
return callback_list
def _cosine_anneal_schedule(self, t):
cos_inner = np.pi *(t %(self.T // self.M))
cos_inner /= self.T // self.M
cos_out = np.cos(cos_inner)+ 1
return float(self.alpha_zero / 2 * cos_out )<define_search_model>
test_preds = learn.get_preds(DatasetType.Test)
 test_df.attribute_ids = join_preds(test_preds, best_thr)
 test_df.head()
<choose_model_class><EOS> 
test_df.to_csv('submission.csv', index=False )