Kaggle competition: TGS Salt Identification Challenge
<SOS> metric: IntersectionOverUnionObjectSegmentation Kaggle data source: tgs-salt-identification-challenge<prepare_output>
plt.style.use('seaborn-white')
 sns.set_style("white")
 %matplotlib inline
my_new_model.evaluate_generator(generator=valid_generator )<predict_on_test>
img_size_ori = 101
img_size_target = 101
im_width = 101
im_height = 101
im_chan = 1
basicpath = '.. /input/'
path_train = basicpath + 'train/'
path_test = basicpath + 'test/'
path_train_images = path_train + 'images/'
path_train_masks = path_train + 'masks/'
path_test_images = path_test + 'images/'
test_generator.reset()
pred=my_new_model.predict_generator(test_generator,verbose=1 )<save_to_csv>
train_df = pd.read_csv(".. /input/train.csv", index_col="id", usecols=[0])
 depths_df = pd.read_csv(".. /input/depths.csv", index_col="id")
 train_df = train_df.join(depths_df)
 test_df = depths_df[~depths_df.index.isin(train_df.index)]
len(train_df )
labels =(train_generator.class_indices)
 labels = list(labels.keys())
 df = pd.DataFrame(data=pred,
columns=labels)
 columns = list(df)
 columns.sort()
df = df.reindex(columns=columns)
 filenames = testdf["id"]
df["id"] = filenames
cols = df.columns.tolist()
cols = cols[-1:] + cols[:-1]
df = df[cols]
df.head(5)
 df.to_csv("results_tl_01.csv",index=False )<set_options>
train_df["images"] = [np.array(load_img(".. /input/train/images/{}.png".format(idx), grayscale=True)) / 255 for idx in tqdm_notebook(train_df.index)]
%reload_ext autoreload
%autoreload 2
%matplotlib inline<import_modules>
train_df["masks"] = [np.array(load_img(".. /input/train/masks/{}.png".format(idx), grayscale=True)) / 255 for idx in tqdm_notebook(train_df.index)]
from fastai.imports import *
from fastai.torch_imports import *
from fastai.transforms import *
from fastai.conv_learner import *
from fastai.model import *
from fastai.dataset import *
from fastai.sgdr import *
from fastai.plots import *<set_options>
train_df["coverage"] = train_df.masks.map(np.sum)/ pow(img_size_ori, 2)
 def cov_to_class(val):
for i in range(0, 11):
if val * 10 <= i :
return i
train_df["coverage_class"] = train_df.coverage.map(cov_to_class)
torch.cuda.set_device(0 )<set_options>
ids_train, ids_valid, x_train, x_valid, y_train, y_valid, cov_train, cov_test, depth_train, depth_test = train_test_split(
 train_df.index.values,
np.array(train_df.images.tolist() ).reshape(-1, img_size_target, img_size_target, 1),
np.array(train_df.masks.tolist() ).reshape(-1, img_size_target, img_size_target, 1),
train_df.coverage.values,
train_df.z.values,
test_size=0.2, stratify=train_df.coverage_class, random_state= 1234 )
torch.backends.cudnn.enabled<define_variables>
ACTIVATION = "relu"
def convolution_block(x, filters, size, strides=(1,1), padding='same', activation=True, bn=True):
x = Conv2D(filters, size, strides=strides, padding=padding )(x)
 if bn == True:
x = BatchNormalization()(x)
 if activation == True:
x = Activation(ACTIVATION )(x)
 return x
def residual_block(blockInput, num_filters=16):
x = BatchNormalization()(blockInput)
 x = Activation(ACTIVATION )(x)
 x = convolution_block(x, num_filters,(3,3))
x = convolution_block(x, num_filters,(3,3), activation=False, bn=False)
 x = Add()([x, blockInput])
 return x
PATH = '.. /input/'
sz = 224
arch = resnet101
bs = 128<load_from_csv>
def build_model(input_layer, start_neurons, DropoutRatio = 0.5):
conv1 = Conv2D(start_neurons * 1,(3, 3), activation=None, padding="same" )(input_layer)
 conv1 = residual_block(conv1,start_neurons * 1)
 conv1 = residual_block(conv1,start_neurons * 1)
 conv1 = Activation(ACTIVATION )(conv1)
 pool1 = MaxPooling2D(( 2, 2))(conv1)
 pool1 = Dropout(DropoutRatio/2 )(pool1)
 conv2 = Conv2D(start_neurons * 2,(3, 3), activation=None, padding="same" )(pool1)
 conv2 = residual_block(conv2,start_neurons * 2)
 conv2 = residual_block(conv2,start_neurons * 2)
 conv2 = Activation(ACTIVATION )(conv2)
 pool2 = MaxPooling2D(( 2, 2))(conv2)
 pool2 = Dropout(DropoutRatio )(pool2)
 conv3 = Conv2D(start_neurons * 4,(3, 3), activation=None, padding="same" )(pool2)
 conv3 = residual_block(conv3,start_neurons * 4)
 conv3 = residual_block(conv3,start_neurons * 4)
 conv3 = Activation(ACTIVATION )(conv3)
 pool3 = MaxPooling2D(( 2, 2))(conv3)
 pool3 = Dropout(DropoutRatio )(pool3)
 conv4 = Conv2D(start_neurons * 8,(3, 3), activation=None, padding="same" )(pool3)
 conv4 = residual_block(conv4,start_neurons * 8)
 conv4 = residual_block(conv4,start_neurons * 8)
 conv4 = Activation(ACTIVATION )(conv4)
 pool4 = MaxPooling2D(( 2, 2))(conv4)
 pool4 = Dropout(DropoutRatio )(pool4)
 convm = Conv2D(start_neurons * 16,(3, 3), activation=None, padding="same" )(pool4)
 convm = residual_block(convm,start_neurons * 16)
 convm = residual_block(convm,start_neurons * 16)
 convm = Activation(ACTIVATION )(convm)
 deconv4 = Conv2DTranspose(start_neurons * 8,(3, 3), strides=(2, 2), padding="same" )(convm)
 uconv4 = concatenate([deconv4, conv4])
 uconv4 = Dropout(DropoutRatio )(uconv4)
 uconv4 = Conv2D(start_neurons * 8,(3, 3), activation=None, padding="same" )(uconv4)
 uconv4 = residual_block(uconv4,start_neurons * 8)
 uconv4 = residual_block(uconv4,start_neurons * 8)
 uconv4 = Activation(ACTIVATION )(uconv4)
 deconv3 = Conv2DTranspose(start_neurons * 4,(3, 3), strides=(2, 2), padding="valid" )(uconv4)
 uconv3 = concatenate([deconv3, conv3])
 uconv3 = Dropout(DropoutRatio )(uconv3)
 uconv3 = Conv2D(start_neurons * 4,(3, 3), activation=None, padding="same" )(uconv3)
 uconv3 = residual_block(uconv3,start_neurons * 4)
 uconv3 = residual_block(uconv3,start_neurons * 4)
 uconv3 = Activation(ACTIVATION )(uconv3)
 deconv2 = Conv2DTranspose(start_neurons * 2,(3, 3), strides=(2, 2), padding="same" )(uconv3)
 uconv2 = concatenate([deconv2, conv2])
 uconv2 = Dropout(DropoutRatio )(uconv2)
 uconv2 = Conv2D(start_neurons * 2,(3, 3), activation=None, padding="same" )(uconv2)
 uconv2 = residual_block(uconv2,start_neurons * 2)
 uconv2 = residual_block(uconv2,start_neurons * 2)
 uconv2 = Activation(ACTIVATION )(uconv2)
 deconv1 = Conv2DTranspose(start_neurons * 1,(3, 3), strides=(2, 2), padding="valid" )(uconv2)
 uconv1 = concatenate([deconv1, conv1])
 uconv1 = Dropout(DropoutRatio )(uconv1)
 uconv1 = Conv2D(start_neurons * 1,(3, 3), activation=None, padding="same" )(uconv1)
 uconv1 = residual_block(uconv1,start_neurons * 1)
 uconv1 = residual_block(uconv1,start_neurons * 1)
 uconv1 = Activation(ACTIVATION )(uconv1)
 uconv1 = Dropout(DropoutRatio/2 )(uconv1)
 output_layer = Conv2D(1,(1,1), padding="same", activation="sigmoid" )(uconv1)
 return output_layer
label_csv = f'{PATH}labels.csv'
n = len(list(open(label_csv)))- 1
val_idxs = get_cv_idxs(n )<load_from_csv>
iou_thresholds = np.array([0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, 0.9, 0.95])
 def iou(img_true, img_pred):
i = np.sum(( img_true*img_pred)>0)
 u = np.sum(( img_true + img_pred)>0)
 if u == 0:
return u
return i/u
def iou_metric(imgs_true, imgs_pred):
num_images = len(imgs_true)
 scores = np.zeros(num_images)
 for i in range(num_images):
if imgs_true[i].sum() == imgs_pred[i].sum() == 0:
scores[i] = 1
else:
scores[i] =(iou_thresholds <= iou(imgs_true[i], imgs_pred[i])).mean()
return scores.mean()
def get_data(sz,bs):
tfms = tfms_from_model(arch, sz, aug_tfms=transforms_side_on, max_zoom=1.1)
 data = ImageClassifierData.from_csv('.', 'train', label_csv, test_name='test',
val_idxs=val_idxs, suffix='.jpg', tfms=tfms, bs=bs)
 return data if sz > 300 else data.resize(340, 'tmp' )<create_dataframe>
x_train = np.append(x_train, [np.fliplr(x)for x in x_train], axis=0)
 y_train = np.append(y_train, [np.fliplr(x)for x in y_train], axis=0)
 print(x_train.shape)
 print(y_valid.shape )
data = get_data(sz,bs )<load_pretrained>
input_layer = Input(( img_size_target, img_size_target, 1))
output_layer = build_model(input_layer, 16,0.5)
 model = Model(input_layer, output_layer)
 model.compile(loss="binary_crossentropy", optimizer="adam", metrics=["acc"] )
learn = ConvLearner.pretrained(arch,data,precompute=True )<find_best_params>
early_stopping = EarlyStopping(monitor='val_acc', mode = 'max',patience=20, verbose=1)
 model_checkpoint = ModelCheckpoint("./unet_best1.model",monitor='val_acc',
mode = 'max', save_best_only=True, verbose=1)
 reduce_lr = ReduceLROnPlateau(monitor='val_acc', mode = 'max',factor=0.5, patience=6, min_lr=0.001, verbose=1)
 epochs = 200
batch_size = 32
history = model.fit(x_train, y_train,
validation_data=[x_valid, y_valid],
epochs=epochs,
batch_size=batch_size,
callbacks=[early_stopping, model_checkpoint, reduce_lr],
verbose=1 )
learning_rate = learn.lr_find()
learn.sched.plot()<train_model>
model = load_model("./unet_best1.model" )
learn.fit(1e-2, 5 )<import_modules>
def predict_result(model,x_test,img_size_target):
preds_test = model.predict(x_test ).reshape(-1, img_size_target, img_size_target)
 preds_test += np.array([ np.fliplr(a)for a in model.predict(np.array([np.fliplr(x)for x in x_test])).reshape(-1, img_size_target, img_size_target)])
 return preds_test/2.0
from sklearn import metrics<feature_engineering>
preds_valid = predict_result(model,x_valid,img_size_target )
log_preds, y = learn.TTA(is_test=True)
 probs = np.mean(np.exp(log_preds),0 )<create_dataframe>
def filter_image(img):
if img.sum() < 100:
return np.zeros(img.shape)
 else:
return img
thresholds = np.linspace(0.3, 0.7, 31)
 ious = np.array([iou_metric(y_valid.reshape(( -1, img_size_target, img_size_target)) , [filter_image(img)for img in preds_valid > threshold])for threshold in tqdm_notebook(thresholds)])
df = pd.DataFrame(probs)
 df.columns = data.classes<prepare_output>
def rle_encode(im):
pixels = im.flatten(order = 'F')
 pixels = np.concatenate([[0], pixels, [0]])
 runs = np.where(pixels[1:] != pixels[:-1])[0] + 1
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs )
df.insert(0, 'id', [o[5:-4] for o in data.test_ds.fnames])
 df.head()<save_to_csv>
del x_train, x_valid, y_train, y_valid, preds_valid
gc.collect()
df.to_csv('submission.csv', index=False )<set_options>
x_test = np.array([(np.array(load_img(".. /input/test/images/{}.png".format(idx), grayscale = True)))/ 255 for idx in tqdm_notebook(test_df.index)] ).reshape(-1, img_size_target, img_size_target, 1)
 preds_test = predict_result(model,x_test,img_size_target )
%reload_ext autoreload
%autoreload 2
%matplotlib inline<import_modules>
t1 = time.time()
pred_dict = {idx: rle_encode(filter_image(preds_test[i] > threshold_best)) for i, idx in enumerate(tqdm_notebook(test_df.index.values)) }
t2 = time.time()
print(f"Usedtime = {t2-t1} s" )
<set_options><EOS> 
sub = pd.DataFrame.from_dict(pred_dict,orient='index')
 sub.index.names = ['id']
sub.columns = ['rle_mask']
sub.to_csv('submission.csv' )