Kaggle competition: Airbus Ship Detection Challenge
<SOS> metric: IntersectionOverUnionObjectSegmentationBeta Kaggle data source: airbus-ship-detection<feature_engineering>
from fastai.conv_learner import *
from fastai.dataset import *
import pandas as pd
import numpy as np
import os
from PIL import Image
from sklearn.model_selection import train_test_split
from tqdm import tnrange, tqdm_notebook
from scipy import ndimage
data['datetime'] = pd.to_datetime(data['datetime'])
 data['hour'] = data['datetime'].dt.hour
data['month'] = data['datetime'].dt.month
test['datetime'] = pd.to_datetime(test['datetime'])
 test['hour'] = data['datetime'].dt.hour
test['month'] = data['datetime'].dt.month<data_type_conversions>
PATH = './'
TRAIN = '.. /input/airbus-ship-detection/train/'
TEST = '.. /input/airbus-ship-detection/test/'
SEGMENTATION = '.. /input/airbus-ship-detection/train_ship_segmentations.csv'
PRETRAINED_DETECTION_PATH = '.. /input/fine-tuning-resnet34-on-ship-detection/models/'
PRETRAINED_SEGMENTATION_PATH = '.. /input/unet34-dice-0-87/models/'
DETECTION_TEST_PRED = '.. /input/fine-tuning-resnet34-on-ship-detection/ship_detection.csv'
exclude_list = ['6384c3e78.jpg','13703f040.jpg', '14715c06d.jpg', '33e0ff2d5.jpg',
'4d4e09f2a.jpg', '877691df8.jpg', '8b909bb20.jpg', 'a8d99130e.jpg',
'ad55c3143.jpg', 'c8260c541.jpg', 'd6c7f17c7.jpg', 'dc3e7c901.jpg',
'e44dffe88.jpg', 'ef87bad36.jpg', 'f083256d8.jpg']
data['season'] = data.season.astype('category')
 data['month'] = data.month.astype('category')
 data['hour'] = data.hour.astype('category')
 data['holiday'] = data.holiday.astype('category')
 data['workingday'] = data.workingday.astype('category')
 data['weather'] = data.weather.astype('category')
 test['season'] = test.season.astype('category')
 test['month'] = test.month.astype('category')
 test['hour'] = test.hour.astype('category')
 test['holiday'] = test.holiday.astype('category')
 test['workingday'] = test.workingday.astype('category')
 test['weather'] = test.weather.astype('category')
<drop_column>
nw = 2
arch = resnet34
data.drop('datetime',axis=1,inplace=True)
 test.drop('datetime',axis=1,inplace=True )<filter>
train_names = [f for f in os.listdir(TRAIN)]
test_names = [f for f in os.listdir(TEST)]
for el in exclude_list:
if(el in train_names): train_names.remove(el)
 if(el in test_names): test_names.remove(el)
 tr_n, val_n = train_test_split(train_names, test_size=0.05, random_state=42)
 segmentation_df = pd.read_csv(os.path.join(PATH, SEGMENTATION)).set_index('ImageId' )
dataWithoutOutliers = data[np.abs(data["count"]-data["count"].mean())<=(3*data["count"].std())] 
<load_from_csv>
def cut_empty(names):
return [name for name in names
if(type(segmentation_df.loc[name]['EncodedPixels'])!= float)]
tr_n_cut = cut_empty(tr_n)
 val_n_cut = cut_empty(val_n )
dataTrain = pd.read_csv(".. /input/train.csv")
 dataTest = pd.read_csv(".. /input/test.csv")
 MergeData = dataTrain.append(dataTest)
 MergeData.reset_index(inplace=True)
 MergeData.drop('index',inplace=True,axis=1 )<feature_engineering>
def get_mask(img_id, df):
shape =(768,768)
 img = np.zeros(shape[0]*shape[1], dtype=np.uint8)
 masks = df.loc[img_id]['EncodedPixels']
if(type(masks)== float): return img.reshape(shape)
 if(type(masks)== str): masks = [masks]
for mask in masks:
s = mask.split()
for i in range(len(s)//2):
start = int(s[2*i])- 1
length = int(s[2*i+1])
 img[start:start+length] = 1
return img.reshape(shape ).T
pd.options.mode.chained_assignment = None
warnings.filterwarnings("ignore", category=DeprecationWarning)
 %matplotlib inline
MergeData["date"] = MergeData.datetime.apply(lambda x : x.split() [0])
 MergeData["hour"] = MergeData.datetime.apply(lambda x : x.split() [1].split(":")[0] ).astype("int")
 MergeData["year"] = MergeData.datetime.apply(lambda x : x.split() [0].split("-")[0])
 MergeData["weekday"] = MergeData.date.apply(lambda dateString : datetime.strptime(dateString,"%Y-%m-%d" ).weekday())
 MergeData["month"] = MergeData.date.apply(lambda dateString : datetime.strptime(dateString,"%Y-%m-%d" ).month)
<choose_model_class>
class pdFilesDataset(FilesDataset):
def __init__(self, fnames, path, transform):
self.segmentation_df = pd.read_csv(SEGMENTATION ).set_index('ImageId')
 super().__init__(fnames, transform, path)
 def get_x(self, i):
img = open_image(os.path.join(self.path, self.fnames[i]))
if self.sz == 768: return img
else: return cv2.resize(img,(self.sz, self.sz))
def get_y(self, i):
mask = np.zeros(( 768,768), dtype=np.uint8)if(self.path == TEST)\
else get_mask(self.fnames[i], self.segmentation_df)
 img = Image.fromarray(mask ).resize(( self.sz, self.sz)).convert('RGB')
 return np.array(img ).astype(np.float32)
 def get_c(self): return 0
dataWind0 = MergeData[MergeData["windspeed"]==0]
dataWindNot0 = MergeData[MergeData["windspeed"]!=0]
rfg_wind = RandomForestRegressor()
windColumns = ["season","weather","humidity","month","temp","year","atemp"]
rfg_wind.fit(dataWindNot0[windColumns], dataWindNot0["windspeed"])
 wind0Values = rfg_wind.predict(X= dataWind0[windColumns])
 dataWind0["windspeed"] = wind0Values
MergeData = dataWindNot0.append(dataWind0)
 MergeData.reset_index(inplace=True)
 MergeData.drop('index',inplace=True,axis=1 )<define_variables>
def get_data(sz,bs):
tfms = tfms_from_model(arch, sz, crop_type=CropType.NO, tfm_y=TfmType.CLASS)
 tr_names = tr_n if(len(tr_n_cut)%bs == 0)else tr_n[:-(len(tr_n_cut)%bs)]
ds = ImageData.get_ds(pdFilesDataset,(tr_names,TRAIN), 
(val_n_cut,TRAIN), tfms, test=(test_names,TEST))
md = ImageData(PATH, ds, bs, num_workers=nw, classes=None)
 md.is_multi = False
return md
categoricalFeatureNames = ["season","holiday","workingday","weather","weekday","month","year","hour"]
numericalFeatureNames = ["temp","humidity","windspeed","atemp"]
dropFeatures = ['casual',"count","datetime","date","registered"]<data_type_conversions>
cut,lr_cut = model_meta[arch]
for var in categoricalFeatureNames:
MergeData[var] = MergeData[var].astype("category" )<prepare_x_and_y>
def get_base() :
layers = cut_model(arch(True), cut)
 return nn.Sequential(*layers )
train = MergeData[pd.notnull(MergeData['count'])].sort_values(by=["datetime"])
 test = MergeData[~pd.notnull(MergeData['count'])].sort_values(by=["datetime"])
 datetimecol = test["datetime"]
yLabels = train["count"]
yLablesRegistered = train["registered"]
yLablesCasual = train["casual"] 
<drop_column>
class UnetBlock(nn.Module):
def __init__(self, up_in, x_in, n_out):
super().__init__()
up_out = x_out = n_out//2
self.x_conv = nn.Conv2d(x_in, x_out, 1)
 self.tr_conv = nn.ConvTranspose2d(up_in, up_out, 2, stride=2)
 self.bn = nn.BatchNorm2d(n_out)
 def forward(self, up_p, x_p):
up_p = self.tr_conv(up_p)
 x_p = self.x_conv(x_p)
 cat_p = torch.cat([up_p,x_p], dim=1)
 return self.bn(F.relu(cat_p))
class SaveFeatures() :
features=None
def __init__(self, m): self.hook = m.register_forward_hook(self.hook_fn)
 def hook_fn(self, module, input, output): self.features = output
def remove(self): self.hook.remove()
class Unet34(nn.Module):
def __init__(self, rn):
super().__init__()
self.rn = rn
self.sfs = [SaveFeatures(rn[i])for i in [2,4,5,6]]
self.up1 = UnetBlock(512,256,256)
 self.up2 = UnetBlock(256,128,256)
 self.up3 = UnetBlock(256,64,256)
 self.up4 = UnetBlock(256,64,256)
 self.up5 = nn.ConvTranspose2d(256, 1, 2, stride=2)
 def forward(self,x):
x = F.relu(self.rn(x))
x = self.up1(x, self.sfs[3].features)
 x = self.up2(x, self.sfs[2].features)
 x = self.up3(x, self.sfs[1].features)
 x = self.up4(x, self.sfs[0].features)
 x = self.up5(x)
 return x[:,0]
def close(self):
for sf in self.sfs: sf.remove()
class UnetModel() :
def __init__(self,model,name='Unet'):
self.model,self.name = model,name
def get_layer_groups(self, precompute):
lgs = list(split_by_idxs(children(self.model.rn), [lr_cut]))
return lgs + [children(self.model)[1:]]
dataTrain = train.drop(dropFeatures,axis=1)
 dataTest = test.drop(dropFeatures,axis=1 )<compute_test_metric>
def IoU(pred, targs):
pred =(pred > 0.5 ).astype(float)
 intersection =(pred*targs ).sum()
return intersection /(( pred+targs ).sum() - intersection + 1.0 )
def rmsle(y, y_,convertExp=True):
if convertExp:
y = np.exp(y),
y_ = np.exp(y_)
 log1 = np.nan_to_num(np.array([np.log(v + 1)for v in y]))
log2 = np.nan_to_num(np.array([np.log(v + 1)for v in y_]))
calc =(log1 - log2)** 2
return np.sqrt(np.mean(calc)) 
<train_model>
def get_score(pred, true):
n_th = 10
b = 4
thresholds = [0.5 + 0.05*i for i in range(n_th)]
n_masks = len(true)
 n_pred = len(pred)
 ious = []
score = 0
for mask in true:
buf = []
for p in pred: buf.append(IoU(p,mask))
ious.append(buf)
 for t in thresholds:
tp, fp, fn = 0, 0, 0
for i in range(n_masks):
match = False
for j in range(n_pred):
if ious[i][j] > t: match = True
if not match: fn += 1
for j in range(n_pred):
match = False
for i in range(n_masks):
if ious[i][j] > t: match = True
if match: tp += 1
else: fp += 1
score +=(( b+1)*tp)/(( b+1)*tp + b*fn + fp)
 return score/n_th
rfModel = RandomForestRegressor(n_estimators=100)
 yLabelsLog = np.log1p(yLabels)
 rfModel.fit(dataTrain,yLabelsLog)
 preds = rfModel.predict(X= dataTrain)
 print("RMSLE Value For Random Forest: ",rmsle(np.exp(yLabelsLog),np.exp(preds),False)) 
<prepare_output>
def split_mask(mask):
threshold = 0.5
threshold_obj = 8
labled,n_objs = ndimage.label(mask > threshold)
 result = []
for i in range(n_objs):
obj =(labled == i + 1 ).astype(int)
 if(obj.sum() > threshold_obj): result.append(obj)
 return result
[max(0, x)for x in np.exp(predsTest)]<prepare_output>
def get_mask_ind(img_id, df, shape =(768,768)) :
masks = df.loc[img_id]['EncodedPixels']
if(type(masks)== float): return []
if(type(masks)== str): masks = [masks]
result = []
for mask in masks:
img = np.zeros(shape[0]*shape[1], dtype=np.uint8)
 s = mask.split()
for i in range(len(s)//2):
start = int(s[2*i])- 1
length = int(s[2*i+1])
 img[start:start+length] = 1
result.append(img.reshape(shape ).T)
 return result
np.exp(predsTest )<predict_on_test>
class Score_eval() :
def __init__(self):
self.segmentation_df = pd.read_csv(SEGMENTATION ).set_index('ImageId')
 self.score, self.count = 0.0, 0
def put(self,pred,name):
true = get_mask_ind(name, self.segmentation_df)
 self.score += get_score(pred,true)
 self.count += 1
def evaluate(self):
return self.score/self.count
gbm = GradientBoostingRegressor(n_estimators=4000,alpha=0.01);
yLabelsLog = np.log1p(yLabels)
 gbm.fit(dataTrain,yLabelsLog)
 preds = gbm.predict(X= dataTrain)
 print("RMSLE Value For Gradient Boost: ",rmsle(np.exp(yLabelsLog),np.exp(preds),False))<save_to_csv>
m = to_gpu(Unet34(get_base()))
models = UnetModel(m )
submission = pd.DataFrame({
"datetime": datetimecol,
"count": [max(0, x)for x in np.exp(gbmpredsTest)]
})
 submission.to_csv('my_submission_19.csv', index=False)
<load_from_csv>
sz = 768
bs = 8
md = get_data(sz,bs )
train = pd.read_csv('.. /input/labeledTrainData.tsv',header = 0, delimiter = '\t')
 test = pd.read_csv('.. /input/testData.tsv',header = 0, delimiter = '\t' )<define_variables>
learn = ConvLearner(md, models)
 learn.models_path = PRETRAINED_SEGMENTATION_PATH
learn.load('Unet34_768_1')
 learn.models_path = PATH
reviews = train['review']
sentiments = train['sentiment']<define_variables>
def model_pred(learner, dl, F_save):
learner.model.eval() ;
name_list = dl.dataset.fnames
num_batchs = len(dl)
 t = tqdm(iter(dl), leave=False, total=num_batchs)
 count = 0
for x,y in t:
py = to_np(F.sigmoid(learn.model(V(x))))
batch_size = len(py)
 for i in range(batch_size):
F_save(py[i],to_np(y[i]),name_list[count])
 count += 1
reviewsT = test['review']<train_on_grid>
score = Score_eval()
process_pred = lambda yp, y, name : score.put(split_mask(yp),name)
 model_pred(learn, md.val_dl, process_pred)
 print('
',score.evaluate() )
word_vectorizer = TfidfVectorizer(
 sublinear_tf=True,
strip_accents='ascii',
analyzer='word',
stop_words='english',
ngram_range=(1, 1),
max_features=10000)
 word_vectorizer.fit(reviews + reviewsT )<split>
ship_detection = pd.read_csv(DETECTION_TEST_PRED)
 ship_detection.head()
X_train, X_test, y_train, y_test = train_test_split(X, sentiments,test_size=0.25)
 print(X_train.shape, X_test.shape, y_train.shape, y_test.shape )<train_model>
test_names = ship_detection.loc[ship_detection['p_ship'] > 0.5, ['id']]['id'].values.tolist()
test_names_nothing = ship_detection.loc[ship_detection['p_ship'] <= 0.5, ['id']]['id'].values.tolist()
len(test_names), len(test_names_nothing )
model=LogisticRegression(solver='lbfgs')
 model.fit(X_train,y_train )<predict_on_test>
md = get_data(sz,bs)
 learn.set_data(md )
preds=model.predict(X_test)
 print(classification_report(preds,y_test))<predict_on_test>
def decode_mask(mask, shape=(768, 768)) :
pixels = mask.T.flatten()
pixels = np.concatenate([[0], pixels, [0]])
 runs = np.where(pixels[1:] != pixels[:-1])[0] + 1
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs )
preds=model.predict(Xt)
 test['sentiment'] = preds<load_from_csv>
ship_list_dict = []
for name in test_names_nothing:
ship_list_dict.append({'ImageId':name,'EncodedPixels':np.nan} )
pd.read_csv('.. /input/sampleSubmission.csv' )<drop_column>
def enc_test(yp, y, name):
masks = split_mask(yp)
 if(len(masks)== 0):
ship_list_dict.append({'ImageId':name,'EncodedPixels':np.nan})
 for mask in masks:
ship_list_dict.append({'ImageId':name,'EncodedPixels':decode_mask(mask)} )
test = test.drop(['review'],axis = 1 )<save_to_csv>
model_pred(learn, md.test_dl, enc_test)
 pred_df = pd.DataFrame(ship_list_dict)
 pred_df.to_csv('submission.csv', index=False )