Kaggle competition: Airbus Ship Detection Challenge
<SOS> metric: IntersectionOverUnionObjectSegmentationBeta Kaggle data source: airbus-ship-detection<import_modules>
from fastai.conv_learner import *
from fastai.dataset import *
import pandas as pd
import numpy as np
import os
from PIL import Image
from sklearn.model_selection import train_test_split
from tqdm import tnrange, tqdm_notebook
from scipy import ndimage
from keras.backend import tf as ktf
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from fastai.vision import *
from fastai.metrics import *
from fastai import *
from os import *
import matplotlib.pyplot as plt
import matplotlib.patches as patches
import random
from sklearn.utils import shuffle<define_variables>
PATH = './'
TRAIN = '.. /input/airbus-ship-detection/train_v2/'
TEST = '.. /input/airbus-ship-detection/test_v2/'
SEGMENTATION = '.. /input/airbus-ship-detection/train_ship_segmentations_v2.csv'
PRETRAINED_DETECTION_PATH = '.. /input/fine-tuning-resnet34-on-ship-detection/models/'
PRETRAINED_SEGMENTATION_PATH = '.. /input/unet34-dice-0-87/models/'
DETECTION_TEST_PRED = '.. /input/fine-tuning-resnet34-ship-redo/Have_ship_or_not.csv'
model_path='.'
path='.. /input/'
train_folder=f'{path}train'
test_folder=f'{path}test'
train_lbl=f'{path}train_labels.csv'
ORG_SIZE=96
bs=64
num_workers=None
sz=96<load_from_csv>
nw = 2
arch = resnet34
df_train=pd.read_csv(train_lbl )<feature_engineering>
train_names = [f for f in os.listdir(TRAIN)][:1000]
test_names = [f for f in os.listdir(TEST)][:1000]
tr_n, val_n = train_test_split(train_names, test_size=0.05, random_state=42)
 segmentation_df = pd.read_csv(os.path.join(PATH, SEGMENTATION)).set_index('ImageId')
 def cut_empty(names):
return [name for name in names
if(type(segmentation_df.loc[name]['EncodedPixels'])!= float)]
tr_n_cut = cut_empty(tr_n)
 val_n_cut = cut_empty(val_n )
tfms = get_transforms(do_flip=True, flip_vert=True, max_rotate=.0, max_zoom=1.1,
max_lighting=0.05, max_warp=0.)<load_from_csv>
def get_mask(img_id, df):
shape =(768,768)
 img = np.zeros(shape[0]*shape[1], dtype=np.uint8)
 masks = df.loc[img_id]['EncodedPixels']
if(type(masks)== float): return img.reshape(shape)
 if(type(masks)== str): masks = [masks]
for mask in masks:
s = mask.split()
for i in range(len(s)//2):
start = int(s[2*i])- 1
length = int(s[2*i+1])
 img[start:start+length] = 1
return img.reshape(shape ).T
data = ImageDataBunch.from_csv(path,csv_labels=train_lbl,folder='train',valid_pct=0.2, ds_tfms=tfms, size=sz, suffix='.tif',test=test_folder,bs=bs);
stats=data.batch_stats()
data.normalize(stats )<define_variables>
class pdFilesDataset(FilesDataset):
def __init__(self, fnames, path, transform):
self.segmentation_df = pd.read_csv(SEGMENTATION ).set_index('ImageId')
 super().__init__(fnames, transform, path)
 def get_x(self, i):
img = open_image(os.path.join(self.path, self.fnames[i]))
if self.sz == 768: return img
else: return cv2.resize(img,(self.sz, self.sz))
def get_y(self, i):
mask = np.zeros(( 768,768), dtype=np.uint8)if(self.path == TEST)\
else get_mask(self.fnames[i], self.segmentation_df)
 img = Image.fromarray(mask ).resize(( self.sz, self.sz)).convert('RGB')
 return np.array(img ).astype(np.float32)
 def get_c(self): return 0
data.show_batch(rows=5, figsize=(12,9))<import_modules>
def get_data(sz,bs):
tfms = tfms_from_model(arch, sz, crop_type=CropType.NO, tfm_y=TfmType.CLASS)
 tr_names = tr_n if(len(tr_n_cut)%bs == 0)else tr_n[:-(len(tr_n_cut)%bs)]
ds = ImageData.get_ds(pdFilesDataset,(tr_names,TRAIN), 
(val_n_cut,TRAIN), tfms, test=(test_names,TEST))
md = ImageData(PATH, ds, bs, num_workers=nw, classes=None)
 return md
from sklearn.metrics import roc_auc_score<compute_test_metric>
cut,lr_cut = model_meta[arch]
def auc_score(y_pred,y_true,tens=True):
score=roc_auc_score(y_true,torch.sigmoid(y_pred)[:,1])
 if tens:
score=tensor(score)
 else:
score=score
return score<choose_model_class>
def get_base(pre=True):
layers = cut_model(arch(pre), cut)
 return nn.Sequential(*layers )
learn = create_cnn(data, models.resnet34, metrics=[auc_score], model_dir="/tmp/model/", ps=0.5 )<train_model>
class UnetBlock(nn.Module):
def __init__(self, up_in, x_in, n_out):
super().__init__()
up_out = x_out = n_out//2
self.x_conv = nn.Conv2d(x_in, x_out, 1)
 self.tr_conv = nn.ConvTranspose2d(up_in, up_out, 2, stride=2)
 self.bn = nn.BatchNorm2d(n_out)
 def forward(self, up_p, x_p):
up_p = self.tr_conv(up_p)
 x_p = self.x_conv(x_p)
 cat_p = torch.cat([up_p,x_p], dim=1)
 return self.bn(F.relu(cat_p))
class SaveFeatures() :
features=None
def __init__(self, m): self.hook = m.register_forward_hook(self.hook_fn)
 def hook_fn(self, module, input, output): self.features = output
def remove(self): self.hook.remove()
class Unet34(nn.Module):
def __init__(self, rn):
super().__init__()
self.rn = rn
self.sfs = [SaveFeatures(rn[i])for i in [2,4,5,6]]
self.up1 = UnetBlock(512,256,256)
 self.up2 = UnetBlock(256,128,256)
 self.up3 = UnetBlock(256,64,256)
 self.up4 = UnetBlock(256,64,256)
 self.up5 = nn.ConvTranspose2d(256, 1, 2, stride=2)
 def forward(self,x):
x = F.relu(self.rn(x))
x = self.up1(x, self.sfs[3].features)
 x = self.up2(x, self.sfs[2].features)
 x = self.up3(x, self.sfs[1].features)
 x = self.up4(x, self.sfs[0].features)
 x = self.up5(x)
 return x[:,0]
def close(self):
for sf in self.sfs: sf.remove()
class UnetModel() :
def __init__(self,model,name='Unet'):
self.model,self.name = model,name
def get_layer_groups(self, precompute):
lgs = list(split_by_idxs(children(self.model.rn), [lr_cut]))
return lgs + [children(self.model)[1:]]
learn.fit_one_cycle(1, 1e-3 )<find_best_params>
def split_mask(mask):
threshold = 0.57777778
threshold_obj = 30
labled,n_objs = ndimage.label(mask > threshold)
 result = []
for i in range(n_objs):
obj =(labled == i + 1 ).astype(int)
 if(obj.sum() > threshold_obj): result.append(obj)
 return result
def get_mask_ind(img_id, df, shape =(768,768)) :
masks = df.loc[img_id]['EncodedPixels']
if(type(masks)== float): return []
if(type(masks)== str): masks = [masks]
result = []
for mask in masks:
img = np.zeros(shape[0]*shape[1], dtype=np.uint8)
 s = mask.split()
for i in range(len(s)//2):
start = int(s[2*i])- 1
length = int(s[2*i+1])
 img[start:start+length] = 1
result.append(img.reshape(shape ).T)
 return result
learn.lr_find()<train_model>
def aug_unit(x,fwd=True,mask=False):
return x
def aug_flipV(x,fwd=True,mask=False):
return x.flip(2)if mask else x.flip(3)
 def aug_flipH(x,fwd=True,mask=False):
return x.flip(1)if mask else x.flip(2)
 def aug_T(x,fwd=True,mask=False):
return torch.transpose(x,1,2)if mask else torch.transpose(x,2,3)
 def aug_rot_2(x,fwd=True,mask=False):
return aug_flipV(aug_flipH(x,fwd,mask),fwd,mask)
 def aug_rot_4cr(x,fwd=True,mask=False):
return aug_flipV(aug_T(x,fwd,mask),fwd,mask)if fwd else \
aug_T(aug_flipV(x,fwd,mask),fwd,mask)
 def aug_rot_4cw(x,fwd=True,mask=False):
return aug_flipH(aug_T(x,fwd,mask),fwd,mask)if fwd else \
aug_T(aug_flipH(x,fwd,mask),fwd,mask)
 def aug_rot_2T(x,fwd=True,mask=False):
return aug_rot_2(aug_T(x,fwd,mask),fwd,mask)
 trms_side_on = [aug_unit,aug_flipH]
trms_top_down = [aug_unit,aug_flipV]
trms_dihedral = [aug_unit,aug_flipH,aug_flipV,aug_T,aug_rot_2,aug_rot_2T,
aug_rot_4cw,aug_rot_4cr]
def enc_img(img):
return torch.transpose(torch.tensor(img),0,2 ).unsqueeze(0)
 def dec_img(img):
return to_np(torch.transpose(img.squeeze(0),0,2))
def display_augs(x,augs=aug_unit):
columns = 4
n = len(augs)
 rows = n//4 + 1
fig=plt.figure(figsize=(columns*4, rows*4))
img = enc_img(x)
 for i in range(rows):
for j in range(columns):
idx = j+i*columns
if idx >= n: break
fig.add_subplot(rows, columns, idx+1)
 plt.axis('off')
 plt.imshow(dec_img(augs[idx](img)))
 plt.show()
img = np.array(Image.open(os.path.join(TRAIN,'ce69faa4b.jpg')))
 display_augs(img,trms_dihedral )
learn.fit_one_cycle(5, slice(1e-4,1e-1))<save_model>
m = to_gpu(Unet34(get_base(False)))
 models = UnetModel(m)
 sz = 768
bs = 1
md = get_data(sz,bs)
 learn = ConvLearner(md, models)
 learn.models_path = PRETRAINED_SEGMENTATION_PATH
learn.load('Unet34_768_1')
 learn.models_path = PATH
learn.save('stage-1' )<find_best_params>
config = ktf.ConfigProto()
config.gpu_options.per_process_gpu_memory_fraction = 0.4
config.gpu_options.visible_device_list = "0"
set_session(ktf.Session(config=config))
print('ktf limited' )
interp = ClassificationInterpretation.from_learner(learn)
 losses,idxs = interp.top_losses()
len(data.valid_ds)==len(losses)==len(idxs )<train_model>
montage_rgb = lambda x: np.stack([montage(x[:, :, :, i])for i in range(x.shape[3])], -1)
 ship_dir = '.. /input/airbus-ship-detection/'
train_image_dir = os.path.join(ship_dir, 'train_v2')
 test_image_dir = os.path.join(ship_dir, 'test_v2')
 exclude_list = ['6384c3e78.jpg','13703f040.jpg', '14715c06d.jpg', '33e0ff2d5.jpg',
'4d4e09f2a.jpg', '877691df8.jpg', '8b909bb20.jpg', 'a8d99130e.jpg',
'ad55c3143.jpg', 'c8260c541.jpg', 'd6c7f17c7.jpg', 'dc3e7c901.jpg',
'e44dffe88.jpg', 'ef87bad36.jpg', 'f083256d8.jpg']
def multi_rle_encode(img, **kwargs):

 labels = label(img)
 if img.ndim > 2:
return [rle_encode(np.sum(labels==k, axis=2), **kwargs)for k in np.unique(labels[labels>0])]
else:
return [rle_encode(labels==k, **kwargs)for k in np.unique(labels[labels>0])]
def rle_encode(img):

 pixels = img.T.flatten()
pixels = np.concatenate([[0], pixels, [0]])
 runs = np.where(pixels[1:] != pixels[:-1])[0] + 1
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs)
 def rle_decode(mask_rle, shape=(768, 768)) :

 s = mask_rle.split()
starts, lengths = [np.asarray(x, dtype=int)for x in(s[0:][::2], s[1:][::2])]
starts -= 1
ends = starts + lengths
img = np.zeros(shape[0]*shape[1], dtype=np.uint8)
 for lo, hi in zip(starts, ends):
img[lo:hi] = 1
return img.reshape(shape ).T
def masks_as_image(in_mask_list):
all_masks = np.zeros(( 768, 768), dtype = np.int16)
 for mask in in_mask_list:
if isinstance(mask, str):
all_masks += rle_decode(mask)
 return np.expand_dims(all_masks, -1 )
learn.fit_one_cycle(1 )<train_model>
masks = pd.read_csv(os.path.join('.. /input/airbus-ship-detection/',
'train_ship_segmentations_v2.csv')).head(5000)
 masks['corrupted'] = masks['ImageId'].map(lambda x: 1 if x in exclude_list else 0)
 print(masks['corrupted'].value_counts())
 masks = masks[masks['corrupted'] == 0]
masks = masks.drop(['corrupted'], axis=1)
 print(masks.shape[0], 'masks found')
 print(masks['ImageId'].value_counts().shape[0])
 masks.head()
masks['ships'] = masks['EncodedPixels'].map(lambda c_row: 1 if isinstance(c_row, str)else 0)
 unique_img_ids = masks.groupby('ImageId' ).agg({'ships': 'sum'} ).reset_index()
unique_img_ids = unique_img_ids[unique_img_ids['ships']>3]
unique_img_ids['has_ship'] = unique_img_ids['ships'].map(lambda x: 1.0 if x>0 else 0.0)
 unique_img_ids['has_ship_vec'] = unique_img_ids['has_ship'].map(lambda x: [x])
 unique_img_ids['file_size_kb'] = unique_img_ids['ImageId'].map(lambda c_img_id:
os.stat(os.path.join(train_image_dir,
c_img_id)).st_size/1024)
 unique_img_ids = unique_img_ids[unique_img_ids['file_size_kb'] > 50]
unique_img_ids['file_size_kb'].hist()
masks.drop(['ships'], axis=1, inplace=True)
 unique_img_ids.sample(7)
 train_ids, valid_ids = train_test_split(unique_img_ids,
test_size = 0.3,
stratify = unique_img_ids['ships'])
 train_df = pd.merge(masks, train_ids)
 valid_df = pd.merge(masks, valid_ids)
 print(train_df.shape[0], 'training masks')
 print(valid_df.shape[0], 'validation masks' )
learn.fit_one_cycle(5, slice(5e-5,5e-3))<save_model>
def masks_as_color(in_mask_list):
all_masks = np.zeros(( 768, 768), dtype = np.float)
 scale = lambda x:(len(in_mask_list)+x+1)/(len(in_mask_list)*2)
 for i,mask in enumerate(in_mask_list):
if isinstance(mask, str):
all_masks[:,:] += scale(i)* rle_decode(mask)
 return all_masks
def raw_prediction(img, path=test_image_dir):
c_img = imread(os.path.join(path, c_img_name))
c_img = np.expand_dims(c_img, 0)/255.0
cur_seg = fullres_model.predict(c_img)[0]
return cur_seg, c_img[0]
def smooth(cur_seg):
return binary_opening(cur_seg>0.57777778, np.expand_dims(disk(2), -1))
def predict(img, path=test_image_dir):
cur_seg, c_img = raw_prediction(img, path=path)
 return smooth(cur_seg), c_img
learn.save('stage-2' )<find_best_params>
fullres_model = models.load_model('.. /input/attention-train3/seg_model.h5', compile=False )
interp = ClassificationInterpretation.from_learner(learn)
 losses,idxs = interp.top_losses()
len(data.valid_ds)==len(losses)==len(idxs )<compute_test_metric>
temptest = []
def get_data(sz,bs = 1):
tfms = tfms_from_model(arch, sz, crop_type=CropType.NO, tfm_y=TfmType.CLASS)
 tr_names = tr_n if(len(tr_n_cut)%bs == 0)else tr_n[:-(len(tr_n_cut)%bs)]
ds = ImageData.get_ds(pdFilesDataset,(tr_names,TRAIN), 
(val_n_cut,TRAIN), tfms ,test=(temptest,TEST))
md = ImageData(PATH, ds, bs, num_workers=nw, classes=None)
 return md
preds,y=learn.get_preds()
pred_score=auc_score(preds,y)
 pred_score<predict_on_test>
have_ship= pd.read_csv(".. /input/fine-tuning-resnet34-ship-redo/ship_detection.csv" )
preds_test,y_test=learn.get_preds(ds_type=DatasetType.Test )<data_type_conversions>
ship_list_dict_64 = []
ship_list_dict_73 = []
ship_list_dict_73_more = []
ship_list_dict_82 = []
ship_list_dict_82_more = []
ship_list_dict_en_64 = []
for name in test_names_nothing:
ship_list_dict_64.append({'ImageId':name,'EncodedPixels':None})
 ship_list_dict_73.append({'ImageId':name,'EncodedPixels':None})
 ship_list_dict_73_more.append({'ImageId':name,'EncodedPixels':None})
 ship_list_dict_82.append({'ImageId':name,'EncodedPixels':None})
 ship_list_dict_82_more.append({'ImageId':name,'EncodedPixels':None})
 len(ship_list_dict_64),len(ship_list_dict_73)
clean_fname=np.vectorize(lambda fname: str(fname ).split('/')[-1].split('.')[0])
 fname_cleaned=clean_fname(data.test_ds.items)
 fname_cleaned=fname_cleaned.astype(str )<save_to_csv>
for c_img_name in tqdm_notebook(test_paths):
if c_img_name in test_names:
c_path = os.path.join(test_image_dir, c_img_name)
 c_img = imread(c_path)
 first_img = np.expand_dims(c_img, 0)/255.0
first_seg = fullres_model.predict(first_img)[0]
temptest = [c_img_name]
md = get_data(768,1)
 learn.set_data(md)
 first_seg2 = model_pred_aug(learn, md.test_dl, trms_dihedral)
 ensemble6_4 = first_seg2*0.6 + first_seg[:, :, 0]*0.4
ensemble7_3 = first_seg2*0.7 + first_seg[:, :, 0]*0.3
ensemble7_3_more = ensemble7_3 * 1.05
ensemble6_4_masks = split_mask(ensemble6_4)
 ensemble7_3_masks = split_mask(ensemble7_3)
 ensemble7_3_more_masks = split_mask(ensemble7_3_more)
 if(len(ensemble6_4_masks)== 0):
ship_list_dict_64.append({'ImageId':c_img_name,'EncodedPixels':np.nan})
 for mask in ensemble6_4_masks:
ship_list_dict_64.append({'ImageId':c_img_name,'EncodedPixels':mask})
 if(len(ensemble7_3_masks)== 0):
ship_list_dict_73.append({'ImageId':c_img_name,'EncodedPixels':np.nan})
 for mask in ensemble7_3_masks:
ship_list_dict_73.append({'ImageId':c_img_name,'EncodedPixels':mask})
 ensemble6_4 = None
ensemble7_3 = None
ensemble6_4_masks = None
ensemble7_3_masks = None
gc.collect()
ensemble8_2 = first_seg2*0.8 + first_seg[:, :, 0]*0.2
ensemble8_2_more = ensemble8_2*1.05
ensemble8_2_masks = split_mask(ensemble8_2)
 ensemble8_2_more_masks = split_mask(ensemble8_2_more)
 if(len(ensemble8_2_masks)== 0):
ship_list_dict_82.append({'ImageId':c_img_name,'EncodedPixels':np.nan})
 for mask in ensemble8_2_masks:
ship_list_dict_82.append({'ImageId':c_img_name,'EncodedPixels':mask})
 if(len(ensemble8_2_more_masks)== 0):
ship_list_dict_82_more.append({'ImageId':c_img_name,'EncodedPixels':np.nan})
 for mask in ensemble8_2_more_masks:
ship_list_dict_82_more.append({'ImageId':c_img_name,'EncodedPixels':mask})
 ensemble8_2 = None
ensemble8_2_more = None
ensemble8_2_masks = None
ensemble8_2_more_masks = None
gc.collect()
len(ship_list_dict_64),len(ship_list_dict_73)
sub.loc[fname_cleaned,'label']=to_np(preds_test[:,1])
 sub.to_csv(f'/kaggle/working/submission_{pred_score}.csv' )<set_options>
submission_64 = pd.DataFrame(ship_list_dict_64)[['ImageId', 'EncodedPixels']]
submission_64.to_csv('submission_64.csv', index=False)
 submission_64.sample(3 )
%matplotlib inline
np.random.seed(2)
 sns.set(style='white', context='notebook', palette='deep')
 for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename)) 
<load_from_csv>
submission_73 = pd.DataFrame(ship_list_dict_73)[['ImageId', 'EncodedPixels']]
submission_73.to_csv('submission_73.csv', index=False)
 submission_73.sample(3 )
train_data = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')
 test_data = pd.read_csv('/kaggle/input/digit-recognizer/test.csv' )<prepare_x_and_y>
submission_73_more = pd.DataFrame(ship_list_dict_73_more)[['ImageId', 'EncodedPixels']]
submission_73_more.to_csv('submission_73_more.csv', index=False)
 submission_73_more.sample(3 )
X = train_data.drop('label', axis=1)
 y = train_data['label']<count_values>
submission_82 = pd.DataFrame(ship_list_dict_82)[['ImageId', 'EncodedPixels']]
submission_82.to_csv('submission_82.csv', index=False)
 submission_82.sample(3 )
<count_missing_values><EOS> 
submission_82_more = pd.DataFrame(ship_list_dict_82_more)[['ImageId', 'EncodedPixels']]
submission_82_more.to_csv('submission_82_more.csv', index=False)
 submission_82_more.sample(3 )