Kaggle competition: Airbus Ship Detection Challenge
<SOS> metric: IntersectionOverUnionObjectSegmentationBeta Kaggle data source: airbus-ship-detection<split>
from fastai.conv_learner import *
from fastai.dataset import *
import pandas as pd
import numpy as np
import os
from PIL import Image
from sklearn.model_selection import train_test_split
from tqdm import tnrange, tqdm_notebook
from scipy import ndimage
X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2, random_state=3 )<choose_model_class>
PATH = './'
TRAIN = '.. /input/airbus-ship-detection/train/'
TEST = '.. /input/airbus-ship-detection/test/'
SEGMENTATION = '.. /input/airbus-ship-detection/train_ship_segmentations.csv'
PRETRAINED_DETECTION_PATH = '.. /input/fine-tuning-resnet34-on-ship-detection/models/'
PRETRAINED_SEGMENTATION_PATH = '.. /input/unet34-dice-0-87/models/'
DETECTION_TEST_PRED = '.. /input/fine-tuning-resnet34-on-ship-detection/ship_detection.csv'
exclude_list = ['6384c3e78.jpg','13703f040.jpg', '14715c06d.jpg', '33e0ff2d5.jpg',
'4d4e09f2a.jpg', '877691df8.jpg', '8b909bb20.jpg', 'a8d99130e.jpg',
'ad55c3143.jpg', 'c8260c541.jpg', 'd6c7f17c7.jpg', 'dc3e7c901.jpg',
'e44dffe88.jpg', 'ef87bad36.jpg', 'f083256d8.jpg']
gbr = GradientBoostingRegressor(random_state=3 )<train_on_grid>
nw = 2
arch = resnet34
params = {'learning_rate': [0.1, 0.01, 0.001, 0.5, 0.05],
'n_estimators': [100, 200, 300, 400]}
grid = GridSearchCV(gbr, param_grid=params, cv=5)
 grid.fit(x, y )<find_best_params>
train_names = [f for f in os.listdir(TRAIN)]
test_names = [f for f in os.listdir(TEST)]
for el in exclude_list:
if(el in train_names): train_names.remove(el)
 if(el in test_names): test_names.remove(el)
 tr_n, val_n = train_test_split(train_names, test_size=0.05, random_state=42)
 segmentation_df = pd.read_csv(os.path.join(PATH, SEGMENTATION)).set_index('ImageId' )
grid.best_params_<compute_train_metric>
def cut_empty(names):
return [name for name in names
if(type(segmentation_df.loc[name]['EncodedPixels'])!= float)]
tr_n_cut = cut_empty(tr_n)
 val_n_cut = cut_empty(val_n )
gbr = GradientBoostingRegressor(learning_rate=0.5, n_estimators=400)
 cross_val_score(gbr, x, y, cv=5 ).mean()<compute_train_metric>
def get_mask(img_id, df):
shape =(768,768)
 img = np.zeros(shape[0]*shape[1], dtype=np.uint8)
 masks = df.loc[img_id]['EncodedPixels']
if(type(masks)== float): return img.reshape(shape)
 if(type(masks)== str): masks = [masks]
for mask in masks:
s = mask.split()
for i in range(len(s)//2):
start = int(s[2*i])- 1
length = int(s[2*i+1])
 img[start:start+length] = 1
return img.reshape(shape ).T
gbr.fit(X_train, y_train)
 pre = gbr.predict(X_test)
 mean_squared_error(pre, y_test )<choose_model_class>
class pdFilesDataset(FilesDataset):
def __init__(self, fnames, path, transform):
self.segmentation_df = pd.read_csv(SEGMENTATION ).set_index('ImageId')
 super().__init__(fnames, transform, path)
 def get_x(self, i):
img = open_image(os.path.join(self.path, self.fnames[i]))
if self.sz == 768: return img
else: return cv2.resize(img,(self.sz, self.sz))
def get_y(self, i):
mask = np.zeros(( 768,768), dtype=np.uint8)if(self.path == TEST)\
else get_mask(self.fnames[i], self.segmentation_df)
 img = Image.fromarray(mask ).resize(( self.sz, self.sz)).convert('RGB')
 return np.array(img ).astype(np.float32)
 def get_c(self): return 0
rbf = RandomForestRegressor(n_estimators=222, random_state=50, max_features='sqrt' )<compute_test_metric>
def get_data(sz,bs):
tfms = tfms_from_model(arch, sz, crop_type=CropType.NO, tfm_y=TfmType.CLASS)
 tr_names = tr_n if(len(tr_n_cut)%bs == 0)else tr_n[:-(len(tr_n_cut)%bs)]
ds = ImageData.get_ds(pdFilesDataset,(tr_names,TRAIN), 
(val_n_cut,TRAIN), tfms, test=(test_names,TEST))
md = ImageData(PATH, ds, bs, num_workers=nw, classes=None)
 md.is_multi = False
return md
cross_val_score(rbf, x, y, cv=5 ).mean()<save_to_csv>
cut,lr_cut = model_meta[arch]
gbr.fit(x, y)
 co = gbr.predict(new_test)
 m = []
for i in(np.exp(co)- 1):
n = round(i)
 m.append(n)
 predict = pd.DataFrame({'datetime': test['datetime'], 'count': m})
 predict.to_csv('gbr.csv', index=False )<import_modules>
def get_base() :
layers = cut_model(arch(True), cut)
 return nn.Sequential(*layers )
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import os
import datetime
import calendar<import_modules>
class UnetBlock(nn.Module):
def __init__(self, up_in, x_in, n_out):
super().__init__()
up_out = x_out = n_out//2
self.x_conv = nn.Conv2d(x_in, x_out, 1)
 self.tr_conv = nn.ConvTranspose2d(up_in, up_out, 2, stride=2)
 self.bn = nn.BatchNorm2d(n_out)
 def forward(self, up_p, x_p):
up_p = self.tr_conv(up_p)
 x_p = self.x_conv(x_p)
 cat_p = torch.cat([up_p,x_p], dim=1)
 return self.bn(F.relu(cat_p))
class SaveFeatures() :
features=None
def __init__(self, m): self.hook = m.register_forward_hook(self.hook_fn)
 def hook_fn(self, module, input, output): self.features = output
def remove(self): self.hook.remove()
class Unet34(nn.Module):
def __init__(self, rn):
super().__init__()
self.rn = rn
self.sfs = [SaveFeatures(rn[i])for i in [2,4,5,6]]
self.up1 = UnetBlock(512,256,256)
 self.up2 = UnetBlock(256,128,256)
 self.up3 = UnetBlock(256,64,256)
 self.up4 = UnetBlock(256,64,256)
 self.up5 = nn.ConvTranspose2d(256, 1, 2, stride=2)
 def forward(self,x):
x = F.relu(self.rn(x))
x = self.up1(x, self.sfs[3].features)
 x = self.up2(x, self.sfs[2].features)
 x = self.up3(x, self.sfs[1].features)
 x = self.up4(x, self.sfs[0].features)
 x = self.up5(x)
 return x[:,0]
def close(self):
for sf in self.sfs: sf.remove()
class UnetModel() :
def __init__(self,model,name='Unet'):
self.model,self.name = model,name
def get_layer_groups(self, precompute):
lgs = list(split_by_idxs(children(self.model.rn), [lr_cut]))
return lgs + [children(self.model)[1:]]
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import MinMaxScaler
from sklearn.metrics import mean_squared_error 
<import_modules>
def IoU(pred, targs):
pred =(pred > 0.5 ).astype(float)
 intersection =(pred*targs ).sum()
return intersection /(( pred+targs ).sum() - intersection + 1.0 )
from xgboost import XGBRegressor
from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor<set_options>
def get_score(pred, true):
n_th = 10
b = 4
thresholds = [0.5 + 0.05*i for i in range(n_th)]
n_masks = len(true)
 n_pred = len(pred)
 ious = []
score = 0
for mask in true:
buf = []
for p in pred: buf.append(IoU(p,mask))
ious.append(buf)
 for t in thresholds:
tp, fp, fn = 0, 0, 0
for i in range(n_masks):
match = False
for j in range(n_pred):
if ious[i][j] > t: match = True
if not match: fn += 1
for j in range(n_pred):
match = False
for i in range(n_masks):
if ious[i][j] > t: match = True
if match: tp += 1
else: fp += 1
score +=(( b+1)*tp)/(( b+1)*tp + b*fn + fp)
 return score/n_th
warnings.filterwarnings("ignore" )<load_from_csv>
def split_mask(mask):
threshold = 0.5
threshold_obj = 8
labled,n_objs = ndimage.label(mask > threshold)
 result = []
for i in range(n_objs):
obj =(labled == i + 1 ).astype(int)
 if(obj.sum() > threshold_obj): result.append(obj)
 return result
train_data = pd.read_csv(path_in + 'train.csv', parse_dates = ['datetime'],
index_col='datetime', infer_datetime_format=True)
 test_data = pd.read_csv(path_in + 'test.csv', parse_dates = ['datetime'],
index_col='datetime', infer_datetime_format=True)
 samp_subm = pd.read_csv(path_in+'sampleSubmission.csv', parse_dates = ['datetime'],
index_col='datetime', infer_datetime_format=True )<compute_test_metric>
def get_mask_ind(img_id, df, shape =(768,768)) :
masks = df.loc[img_id]['EncodedPixels']
if(type(masks)== float): return []
if(type(masks)== str): masks = [masks]
result = []
for mask in masks:
img = np.zeros(shape[0]*shape[1], dtype=np.uint8)
 s = mask.split()
for i in range(len(s)//2):
start = int(s[2*i])- 1
length = int(s[2*i+1])
 img[start:start+length] = 1
result.append(img.reshape(shape ).T)
 return result
def rmse(y_true, y_pred):

 return K.sqrt(K.mean(K.square(y_pred - y_true)) )<define_variables>
class Score_eval() :
def __init__(self):
self.segmentation_df = pd.read_csv(SEGMENTATION ).set_index('ImageId')
 self.score, self.count = 0.0, 0
def put(self,pred,name):
true = get_mask_ind(name, self.segmentation_df)
 self.score += get_score(pred,true)
 self.count += 1
def evaluate(self):
return self.score/self.count
num_months_per_year = 12
year_list = [2011, 2012]<feature_engineering>
m = to_gpu(Unet34(get_base()))
models = UnetModel(m )
month = 5
year = 2011
start_date = datetime.datetime(year, month, 1, 0, 0, 0)
 end_date = datetime.datetime(year, month, 19, 23, 0, 0)
<data_type_conversions>
sz = 768
bs = 8
md = get_data(sz,bs )
train_data_temp = pd.DataFrame(columns=train_data.columns)
 for year in year_list:
for month in range(num_months_per_year):
start_date = datetime.datetime(year, month+1, 1, 0, 0, 0)
 end_date = datetime.datetime(year, month+1, 19, 23, 0, 0)
 temp = train_data[start_date:end_date].resample('H' ).asfreq()
features_fill_zero = ['casual', 'registered', 'count']
temp[features_fill_zero] = temp[features_fill_zero].fillna(0)
 features_fill_bbfil = ['season', 'holiday', 'workingday', 'weather']
temp[features_fill_bbfil] = temp[features_fill_bbfil].fillna(method='bfill')
 features_fill_linear = ['temp', 'atemp', 'humidity', 'windspeed']
temp[features_fill_linear] = temp[features_fill_linear].interpolate(method='linear')
 train_data_temp = train_data_temp.append(temp)
 train_data = train_data_temp<feature_engineering>
learn = ConvLearner(md, models)
 learn.models_path = PRETRAINED_SEGMENTATION_PATH
learn.load('Unet34_768_1')
 learn.models_path = PATH
train_data['weekday'] = train_data.index.weekday
train_data['hour'] = train_data.index.hour
test_data['weekday'] = test_data.index.weekday
test_data['hour'] = test_data.index.hour<groupby>
def model_pred(learner, dl, F_save):
learner.model.eval() ;
name_list = dl.dataset.fnames
num_batchs = len(dl)
 t = tqdm(iter(dl), leave=False, total=num_batchs)
 count = 0
for x,y in t:
py = to_np(F.sigmoid(learn.model(V(x))))
batch_size = len(py)
 for i in range(batch_size):
F_save(py[i],to_np(y[i]),name_list[count])
 count += 1
def hour_group(s):
if(( 0<=s)&(s<=6)) :
return 1
elif(( s==7)|(s==9)) :
return 2
elif(( s==8)|(s==16)|(s==19)) :
return 3
elif(( 10<=s)&(s<=15)) :
return 4
elif(( s==17)|(s==18)) :
return 5
elif(20<=s):
return 6<feature_engineering>
score = Score_eval()
process_pred = lambda yp, y, name : score.put(split_mask(yp),name)
 model_pred(learn, md.val_dl, process_pred)
 print('
',score.evaluate() )
train_data['hour_group'] = train_data['hour'].apply(hour_group)
 test_data['hour_group'] = test_data['hour'].apply(hour_group )<feature_engineering>
ship_detection = pd.read_csv(DETECTION_TEST_PRED)
 ship_detection.head()
<data_type_conversions>
test_names = ship_detection.loc[ship_detection['p_ship'] > 0.5, ['id']]['id'].values.tolist()
test_names_nothing = ship_detection.loc[ship_detection['p_ship'] <= 0.5, ['id']]['id'].values.tolist()
len(test_names), len(test_names_nothing )
features_one_hot = ['weekday', 'hour_group', 'weather']
train_data[features_one_hot] = train_data[features_one_hot].astype(int ).astype(str)
 test_data[features_one_hot] = test_data[features_one_hot].astype(int ).astype(str )<categorify>
md = get_data(sz,bs)
 learn.set_data(md )
train_data = pd.get_dummies(train_data)
 test_data = pd.get_dummies(test_data )<define_variables>
def decode_mask(mask, shape=(768, 768)) :
pixels = mask.T.flatten()
pixels = np.concatenate([[0], pixels, [0]])
 runs = np.where(pixels[1:] != pixels[:-1])[0] + 1
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs )
scale_features = ['temp', 'atemp', 'humidity', 'hour', 'windspeed']<normalization>
ship_list_dict = []
for name in test_names_nothing:
ship_list_dict.append({'ImageId':name,'EncodedPixels':np.nan} )
scaler = MinMaxScaler()
train_data[scale_features] = scaler.fit_transform(train_data[scale_features])
 test_data[scale_features] = scaler.transform(test_data[scale_features] )<define_variables>
def enc_test(yp, y, name):
masks = split_mask(yp)
 if(len(masks)== 0):
ship_list_dict.append({'ImageId':name,'EncodedPixels':np.nan})
 for mask in masks:
ship_list_dict.append({'ImageId':name,'EncodedPixels':decode_mask(mask)} )
<prepare_x_and_y><EOS> 
model_pred(learn, md.test_dl, enc_test)
 pred_df = pd.DataFrame(ship_list_dict)
 pred_df.to_csv('submission.csv', index=False )