Kaggle competition: Airbus Ship Detection Challenge
<SOS> metric: IntersectionOverUnionObjectSegmentationBeta Kaggle data source: airbus-ship-detection<feature_engineering>
!pip install -U segmentation-models==0.2.1
!pip install albumentations
all_data["mpaa_rating"] = all_data["mpaa_rating"].apply(lambda s: s.split(" ", 1)[1] if isinstance(s, str)else s)
 all_data["mpaa_rating_reason"] = all_data["mpaa_rating"].apply(lambda s: s.split(" ", 2)[2] if isinstance(s, str)else s)
 all_data["mpaa_rating"] = all_data["mpaa_rating"].apply(lambda s: s.split(" ", 1)[0] if isinstance(s, str)else s)
 all_data["mpaa_rating"].value_counts()<feature_engineering>
import os, sys, gc
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
from tqdm.auto import tqdm
from multiprocessing import Pool, cpu_count
from cv2 import resize
from skimage.io import imread as skiImgRead
from skimage.morphology import label
from sklearn.model_selection import train_test_split
from segmentation_models import Unet
from segmentation_models.backbones import get_preprocessing
from segmentation_models.utils import set_trainable
from segmentation_models.losses import bce_jaccard_loss, bce_dice_loss
from segmentation_models.metrics import iou_score, f2_score
from keras.callbacks import ReduceLROnPlateau, ModelCheckpoint, EarlyStopping, LearningRateScheduler
def SplitReason(r):
s = r.strip().replace(" ", " " ).replace(", and for ", ", " ).replace(", and ", ", ")
 rl = s.rsplit(" and ", 1)
 rl2 = rl[0].split(", ")
 if len(rl)>1:
rl2.append(rl[1])
 return(rl2)
 col_name = "mpaa_rating_reason"
all_data[col_name] = all_data[col_name].apply(lambda s: SplitReason(s)if isinstance(s, str)else [])
 reason_list_list = list(all_data[col_name].values)
 top_reasons = [reason[0] for reason in collections.Counter([i for reason_list in reason_list_list for i in reason_list] ).most_common(100)]
for reason in top_reasons:
all_data[col_name + "_" + reason] = all_data[col_name].apply(lambda x: reason in x)
 all_data.head()
drop_column(col_name )<drop_column>
IMG_HW = 768
ZOOM_HW = 384
DATA_DIR = '.. /input/airbus-ship-detection'
TRAIN_DIR = os.path.join(DATA_DIR, 'train_v2')
 TEST_DIR = os.path.join(DATA_DIR, 'test_v2')
 BACKBONE = 'resnet34'
preprocess_input = get_preprocessing(BACKBONE )
all_data.drop(['imdb_id'], axis=1, inplace=True)
 all_data.drop(['original_title'], axis=1, inplace=True)
 all_data.drop(['overview'], axis=1, inplace=True)
 all_data.drop(['poster_path'], axis=1, inplace=True)
 all_data.drop(['status'], axis=1, inplace=True)
 all_data.drop(['tagline'], axis=1, inplace=True)
 all_data.drop(['title'], axis=1, inplace=True)
 all_data.drop(['Keywords'], axis=1, inplace=True)
 drop_column("imdb_release")
 drop_column("imdb_budget")
 drop_column("imdb_country" )<import_modules>
def rle_decode(rle_mask):

 s = rle_mask.split()
starts, lengths = [np.asarray(x, dtype=int)for x in(s[0:][::2], s[1:][::2])]
starts -= 1
ends = starts + lengths
img = np.zeros(IMG_HW*IMG_HW, dtype=np.uint8)
 for lo, hi in zip(starts, ends):
img[lo:hi] = 1
return img.reshape(IMG_HW,IMG_HW ).T
def rle_encode(im):

 pixels = im.flatten(order = 'F')
 pixels = np.concatenate([[0], pixels, [0]])
 runs = np.where(pixels[1:] != pixels[:-1])[0] + 1
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs )
n_folds = 5
def rmsle_cv(model):
kf = KFold(n_folds, shuffle=True, random_state=42 ).get_n_splits(train.values)
 rmse = np.sqrt(-cross_val_score(model, train.values, y_train, scoring="neg_mean_squared_error", cv=kf))
return(rmse)
 def eval_model(model, name):
start_time = time.time()
score = rmsle_cv(model)
 print("{} score: {:.4f}({:.4f}), execution time: {:.1f}".format(name, score.mean() , score.std() , time.time() -start_time))<choose_model_class>
train_csv = pd.read_csv(os.path.join(DATA_DIR, 'train_ship_segmentations_v2.csv'))
train_csv['withShip'] = ~train_csv['EncodedPixels'].isnull()
train_csv['npixel'] = count_pix_inpool(train_csv['EncodedPixels'] )
mod_lasso = make_pipeline(RobustScaler() , Lasso(alpha=0.005, random_state=1))
eval_model(mod_lasso, "lasso")
 mod_enet = make_pipeline(RobustScaler() , ElasticNet(alpha=0.0005, l1_ratio=.9, random_state=3))
eval_model(mod_enet, "enet")
 mod_cat = CatBoostRegressor(iterations=10000, learning_rate=0.01,
depth=5, eval_metric='RMSE',
colsample_bylevel=0.7, random_seed = 17, silent=True,
bagging_temperature = 0.2, early_stopping_rounds=200)
 eval_model(mod_cat, "cat")
 mod_gboost = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05,
max_depth=5, max_features='sqrt',
min_samples_leaf=15, min_samples_split=10,
loss='huber', random_state=5)
 eval_model(mod_gboost, "gboost")
 mod_xgb = xgb.XGBRegressor(colsample_bytree=0.4603, gamma=0.0468,
learning_rate=0.05, max_depth=3,
min_child_weight=1.7817, n_estimators=2200,
reg_alpha=0.4640, reg_lambda=0.8571,
subsample=0.5213, silent=1,
random_state=7, nthread=-1)
 eval_model(mod_xgb, "xgb")
 mod_lgb = lgb.LGBMRegressor(objective='regression',num_leaves=8,
learning_rate=0.05, n_estimators=650,
max_bin=58, bagging_fraction=0.80,
bagging_freq=5, feature_fraction=0.2319,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf=7, min_sum_hessian_in_leaf=11)
 eval_model(mod_lgb, "lgb" )<train_model>
b_train_csv, b_valid_csv = train_test_split(balanced_train_csv['ImageId'], test_size = 0.2)
 b_train_csv = balanced_train_csv.set_index('ImageId' ).loc[b_train_csv].reset_index()
b_valid_csv = balanced_train_csv.set_index('ImageId' ).loc[b_valid_csv].reset_index()
class StackingAveragedModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, base_models, meta_model, n_folds=5):
self.base_models = base_models
self.meta_model = meta_model
self.n_folds = n_folds
def fit(self, X, y):
self.base_models_ = [list() for x in self.base_models]
self.meta_model_ = clone(self.meta_model)
 kfold = KFold(n_splits=self.n_folds, shuffle=True, random_state=156)
 out_of_fold_predictions = np.zeros(( X.shape[0], len(self.base_models)))
 for i, model in enumerate(self.base_models):
for train_index, holdout_index in kfold.split(X, y):
instance = clone(model)
 self.base_models_[i].append(instance)
 instance.fit(X[train_index], y[train_index])
 y_pred = instance.predict(X[holdout_index])
 out_of_fold_predictions[holdout_index, i] = y_pred
self.meta_model_.fit(out_of_fold_predictions, y)
 return self
def predict(self, X):
meta_features = np.column_stack([
np.column_stack([model.predict(X)for model in base_models] ).mean(axis=1)
 for base_models in self.base_models_ ])
 return self.meta_model_.predict(meta_features)
 mod_stacked = StackingAveragedModels(base_models =(mod_cat, mod_xgb, mod_gboost, mod_lgb), meta_model = mod_lasso)
 eval_model(mod_stacked, "stacked" )<predict_on_test>
def batch_data_gen(csv_df, dir_prefix, batch_size, augmentation=None):
name_idx_df = csv_df.set_index('ImageId')
 img_ids = np.array(name_idx_df.index.unique().tolist())
 n_imgs = img_ids.shape[0]
while True:
np.random.shuffle(img_ids)
 for idx in range(0, n_imgs, batch_size):
batch_x = np.zeros(( batch_size,)+(ZOOM_HW, ZOOM_HW, 3))
batch_y = np.zeros(( batch_size,)+(ZOOM_HW, ZOOM_HW, 1))
end_idx = idx + batch_size
batch_img_ids = img_ids[idx:end_idx]
for i,img_id in enumerate(batch_img_ids):
img_df = name_idx_df.loc[[img_id]]
x, y = load_paired_data(img_df, dir_prefix, augmentation=augmentation)
 batch_x[i] += x
batch_y[i] += y
yield batch_x, batch_y
def load_paired_data(df, dir_prefix, augmentation=None):
img_id = df.index.unique() [0]
try:
image = preprocess_input(skiImgRead(os.path.join(dir_prefix, img_id)))
 except:
image = preprocess_input(np.zeros(( IMG_HW, IMG_HW, 3), dtype=np.uint8))
mask = np.zeros(( IMG_HW, IMG_HW, 1))
for _,mask_rle in df['EncodedPixels'].iteritems() :
if mask_rle is np.nan:
continue
mask[:,:,0] += rle_decode(mask_rle)
 if augmentation:
augmented = augmentation(image=image, mask=mask)
 image = augmented['image']
mask = augmented['mask']
image = resize(image,(ZOOM_HW,ZOOM_HW))
mask = resize(mask.reshape(IMG_HW,IMG_HW),(ZOOM_HW,ZOOM_HW)).reshape(( ZOOM_HW,ZOOM_HW,1))
return image, mask
def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))
def predict(model):
model.fit(train.values, y_train)
 train_pred = model.predict(train.values)
 pred = np.expm1(model.predict(test.values))
print(rmsle(y_train, train_pred))
return(pred )<predict_on_test>
Compose, HorizontalFlip, VerticalFlip, RandomRotate90, ShiftScaleRotate, Transpose,
OneOf, ToFloat,
RandomBrightness, RandomContrast, RandomGamma, CLAHE,
GridDistortion, ElasticTransform, JpegCompression,
RGBShift, GaussNoise, IAAAdditiveGaussianNoise, HueSaturationValue,
Blur, MotionBlur, MedianBlur, RandomBrightnessContrast,
GridDistortion, OpticalDistortion, RandomSizedCrop, CenterCrop 
)
 augmentor = Compose([
OneOf([
HorizontalFlip() ,
VerticalFlip() ,
RandomRotate90() ,
Transpose() ,
], p=0.8),
ShiftScaleRotate(rotate_limit=20),
OneOf([
MotionBlur(blur_limit=3),
MedianBlur(blur_limit=3),
Blur(blur_limit=3),
], p=0.3),
OneOf([
RandomGamma() ,
RandomContrast() ,
RandomBrightness() ,
CLAHE() ,
], p=0.3),
OneOf([
IAAAdditiveGaussianNoise() ,
HueSaturationValue() ,
GaussNoise() ,
], p=0.2),
OneOf([
ElasticTransform() ,
OpticalDistortion() ,
GridDistortion() ,
], p=0.3),
RandomSizedCrop(min_max_height=(IMG_HW/2, IMG_HW), height=IMG_HW, width=IMG_HW, p=0.3),
ToFloat(max_value=1),
],p=1 )
prediction = predict(mod_lasso)
 prediction = predict(mod_enet)
 prediction = predict(mod_xgb)
 prediction = predict(mod_gboost)
 prediction = predict(mod_lgb)
 prediction = predict(mod_stacked )<save_to_csv>
model = Unet(
 BACKBONE,
encoder_weights='imagenet',
classes=1,
activation='sigmoid',
input_shape=(ZOOM_HW, ZOOM_HW, 3),
decoder_filters=(128, 64, 32, 16, 8), 
)
 model.compile(optimizer='Adam', loss=bce_dice_loss, metrics=[iou_score])
submission = pd.DataFrame()
submission['id'] = test_ID
submission['revenue'] = prediction
submission.to_csv('submission.csv', index=False )<set_options>
BATCH_SIZE = 32
%load_ext autoreload
%autoreload 2
%matplotlib inline<import_modules>
checkpoint = ModelCheckpoint(
 filepath='./best_model.h5',
monitor='val_iou_score', mode='max',
save_best_only=True, save_weights_only=True,
verbose=1 
)
 reduce_lr = ReduceLROnPlateau(
 monitor='val_loss', mode='min',
factor=0.3, patience=3, min_lr=0.00001,
verbose=1 
)
from fastai.tabular import *
from sklearn.model_selection import train_test_split, StratifiedKFold, KFold, cross_val_score, GridSearchCV, RepeatedStratifiedKFold
import time
import lightgbm as lgb
import xgboost as xgb
import ast
from sklearn.metrics import mean_squared_error
import seaborn as sns
from sklearn.preprocessing import MultiLabelBinarizer
from sklearn.decomposition import TruncatedSVD<define_variables>
history = model.fit_generator(
 generator = batch_data_gen(b_train_csv, TRAIN_DIR, BATCH_SIZE, augmentation=None),
validation_data = batch_data_gen(b_valid_csv, TRAIN_DIR, BATCH_SIZE),
validation_steps = 50,
steps_per_epoch = 500,
epochs = 20,
verbose = 2,
callbacks=[ checkpoint, reduce_lr ]
)
PATH = ".. /input/tmdb-box-office-prediction/"<load_from_csv>
model.load_weights('best_model.h5' )
train = pd.read_csv(f'{PATH}train.csv', parse_dates=['release_date'])
 test = pd.read_csv(f'{PATH}test.csv', parse_dates=['release_date'] )<load_from_csv>
lrs = LearningRateScheduler(lambda epoch: 0.001 * 0.3 **(epoch // 2))
history = model.fit_generator(
 generator = batch_data_gen(b_train_csv, TRAIN_DIR, BATCH_SIZE, augmentation=augmentor),
validation_data = batch_data_gen(b_valid_csv, TRAIN_DIR, BATCH_SIZE),
validation_steps = 20,
steps_per_epoch = 150,
epochs = 10,
verbose = 2,
callbacks=[ checkpoint, lrs ],
)
train_votes = pd.read_csv('.. /input/tmdb-prediction-votes/trainRatingTotalVotes.csv')
 test_votes = pd.read_csv('.. /input/tmdb-prediction-votes/testRatingTotalVotes.csv')
 train = pd.merge(train, train_votes, how='left', on=['imdb_id'])
 test = pd.merge(test, test_votes, how='left', on=['imdb_id'] )<feature_engineering>
model.load_weights('best_model.h5' )
dict_columns = ['belongs_to_collection', 'genres', 'production_companies',
'production_countries', 'spoken_languages', 'Keywords', 'cast']
def text_to_dict(df):
for column in dict_columns:
df[column] = df[column].apply(lambda x: {} if pd.isna(x)else ast.literal_eval(x))
return df
train = text_to_dict(train)
 test = text_to_dict(test )<define_variables>
fig, axs = plt.subplots(ncols=3, nrows=5, figsize=(10, 15), sharex=True, sharey=True)
 for i, img_id in enumerate(np.random.choice(train_csv.dropna() ['ImageId'].unique() , 5)) :
print(img_id)
 img_df = train_csv.set_index('ImageId' ).loc[[img_id]]
x, y = load_paired_data(img_df, TRAIN_DIR)
 yp = model.predict(np.expand_dims(preprocess_input(x), axis=0))
axs[i,0].set_ylabel(img_id)
 axs[i,0].imshow(x)
 axs[i,1].imshow(y[:,:,0])
 axs[i,2].imshow(yp[0,:,:,0])
 axs[0,0].set_title('Input')
 axs[0,1].set_title('Mask')
 axs[0,2].set_title('Predict')
 plt.xticks([])
 plt.yticks([])
 plt.show()
def build_category_list(x, field, feature):
regex = re.compile('[^0-9a-zA-Z_]')
 category_list = ""
for d in x:
new_category = regex.sub('', d[field].lower().replace(" ","_"))
category_list += new_category + ","
return category_list.strip().strip("," ).split(",")
 target_fields = {'belongs_to_collection': 'name', 'genres': 'name',
'production_countries': 'iso_3166_1', 'production_companies': 'name',
'spoken_languages': 'iso_639_1', 'Keywords': 'name', 'cast': 'name'
}
for k,v in target_fields.items() :
train[k] = train[k].apply(lambda x: build_category_list(x, v, k))
test[k] = test[k].apply(lambda x: build_category_list(x, v, k))
target_fields = {'cast':{'field':'name', 'role_field':'order', 'role_values':[0,1,2]}}<feature_engineering>
sub_csv = pd.DataFrame(columns=['ImageId', 'EncodedPixels'])
 for test_id in tqdm(os.listdir(TEST_DIR)) :
fp = os.path.join(TEST_DIR, test_id)
 img = skiImgRead(fp)
 assert(img.shape ==(IMG_HW, IMG_HW, 3)) , 'Bad Shape in image: "{}"'.format(fp)
 img = resize(img,(ZOOM_HW, ZOOM_HW))
imgTTA1 = preprocess_input(img ).reshape(1, ZOOM_HW, ZOOM_HW, 3)
 imgTTA1 = imgTTA1[:, :: 1, :: 1, :]
imgTTA2 = imgTTA1[:, :: 1, ::-1, :]
imgTTA3 = imgTTA1[:, ::-1, :: 1, :]
imgTTA4 = imgTTA1[:, ::-1, ::-1, :] 
(rTTA1,rTTA2,rTTA3,rTTA4)= model.predict(np.concatenate([imgTTA1, imgTTA2, imgTTA3, imgTTA4])) [:,:,:,0]
result =(
 rTTA1[:: 1, :: 1] +
rTTA2[:: 1, ::-1] +
rTTA3[::-1, :: 1] +
rTTA4[::-1, ::-1] 
)/4
result = resize(result,(IMG_HW, IMG_HW))
labels = label(( result>0.5)+0)
 if labels.max() == 0:
sub_csv = sub_csv.append({'ImageId':test_id}, ignore_index=True)
 else:
for k in np.unique(labels[labels>0]):
sub_csv = sub_csv.append(
 {
'ImageId' : test_id,
'EncodedPixels': rle_encode(labels==k)
 }, ignore_index=True)
<feature_engineering><EOS> 
sub_csv.to_csv('submission.csv', index=False )