Kaggle competition: TGS Salt Identification Challenge
<SOS> metric: IntersectionOverUnionObjectSegmentation Kaggle data source: tgs-salt-identification-challenge<define_variables>
plt.style.use('seaborn-white')
 sns.set_style("white")
CATEGORIES = list(a.class_indices.keys() )<define_variables>
img_size_ori = 101
img_size_target = 128
def upsample_reflect(img):
if img_size_ori == img_size_target:
return img
return np.pad(img[:,:,0],(( 13,14),(13,14)) ,'reflect')
 def upsample(img):
if img_size_ori == img_size_target:
return img
return resize(img,(img_size_target, img_size_target), mode='constant', preserve_range=True)
 def downsample(img):
if img_size_ori == img_size_target:
return img
return resize(img,(img_size_ori, img_size_ori), mode='constant', preserve_range=True)
 def iouMetric(true, pred):
true = K.batch_flatten(true)
 pred = K.batch_flatten(pred)
 pred = K.cast(K.greater(pred, 0.5), K.floatx())
 intersec = true * pred
iou = K.sum(intersec)/(K.sum(true + pred - intersec)+ K.epsilon())
 return iou
R_categories = {y:x for x,y in a.class_indices.items() }
R_categories<set_options>
def dice_coef(y_true, y_pred):
y_true_f = K.flatten(y_true)
 y_pred = K.cast(y_pred, 'float32')
 y_pred_f = K.cast(K.greater(K.flatten(y_pred), 0.5), 'float32')
 intersection = y_true_f * y_pred_f
score = 2.* K.sum(intersection)/(K.sum(y_true_f)+ K.sum(y_pred_f))
return score
def dice_loss(y_true, y_pred):
smooth = 1.
 y_true_f = K.flatten(y_true)
 y_pred_f = K.flatten(y_pred)
 intersection = y_true_f * y_pred_f
score =(2.* K.sum(intersection)+ smooth)/(K.sum(y_true_f)+ K.sum(y_pred_f)+ smooth)
 return 1.- score
def bce_dice_loss(y_true, y_pred):
return binary_crossentropy(y_true, y_pred)+ dice_loss(y_true, y_pred)
 def bce_logdice_loss(y_true, y_pred):
return binary_crossentropy(y_true, y_pred)- K.log(1.- dice_loss(y_true, y_pred))
def weighted_bce_loss(y_true, y_pred, weight):
epsilon = 1e-7
y_pred = K.clip(y_pred, epsilon, 1.- epsilon)
 logit_y_pred = K.log(y_pred /(1.- y_pred))
loss = weight *(logit_y_pred *(1.- y_true)+
K.log(1.+ K.exp(-K.abs(logit_y_pred)))+ K.maximum(-logit_y_pred, 0.))
 return K.sum(loss)/ K.sum(weight)
 def weighted_dice_loss(y_true, y_pred, weight):
smooth = 1.
 w, m1, m2 = weight, y_true, y_pred
intersection =(m1 * m2)
 score =(2.* K.sum(w * intersection)+ smooth)/(K.sum(w * m1)+ K.sum(w * m2)+ smooth)
 loss = 1.- K.sum(score)
 return loss
def weighted_bce_dice_loss(y_true, y_pred):
y_true = K.cast(y_true, 'float32')
 y_pred = K.cast(y_pred, 'float32')
 averaged_mask = K.pool2d(
 y_true, pool_size=(50, 50), strides=(1, 1), padding='same', pool_mode='avg')
 weight = K.ones_like(averaged_mask)
 w0 = K.sum(weight)
 weight = 5.* K.exp(-5.* K.abs(averaged_mask - 0.5))
w1 = K.sum(weight)
 weight *=(w0 / w1)
 loss = weighted_bce_loss(y_true, y_pred, weight)+ dice_loss(y_true, y_pred)
 return loss
del valid
gc.collect()<predict_on_test>
def do_resize2(image, mask, H, W):
image = cv2.resize(image,dsize=(W,H))
mask = cv2.resize(mask,dsize=(W,H))
mask =(mask>0.5 ).astype(np.float32)
 return image,mask
def compute_center_pad(H,W, factor=32):
if H%factor==0:
dy0,dy1=0,0
else:
dy = factor - H%factor
dy0 = dy//2
dy1 = dy - dy0
if W%factor==0:
dx0,dx1=0,0
else:
dx = factor - W%factor
dx0 = dx//2
dx1 = dx - dx0
return dy0, dy1, dx0, dx1
def do_center_pad_to_factor(image, factor=32):
H,W = image.shape[:2]
dy0, dy1, dx0, dx1 = compute_center_pad(H,W, factor)
 image = cv2.copyMakeBorder(image, dy0, dy1, dx0, dx1, cv2.BORDER_REFLECT_101)
 return image
def do_center_pad_to_factor2(image, mask, factor=32):
image = do_center_pad_to_factor(image, factor)
 mask = do_center_pad_to_factor(mask, factor)
 return image, mask
def do_horizontal_flip(image):
image = cv2.flip(image,1)
 return image
def do_horizontal_flip2(image,mask):
image = do_horizontal_flip(image)
 mask = do_horizontal_flip(mask)
 return image, mask
def compute_random_pad(H,W, limit=(-4,4), factor=32):
if H%factor==0:
dy0,dy1=0,0
else:
dy = factor - H%factor
dy0 = dy//2 + np.random.randint(limit[0],limit[1])
 dy1 = dy - dy0
if W%factor==0:
dx0,dx1=0,0
else:
dx = factor - W%factor
dx0 = dx//2 + np.random.randint(limit[0],limit[1])
 dx1 = dx - dx0
return dy0, dy1, dx0, dx1
def do_random_pad_to_factor2(image, mask, limit=(-4,4), factor=32):
H,W = image.shape[:2]
dy0, dy1, dx0, dx1 = compute_random_pad(H,W, limit, factor)
 image = cv2.copyMakeBorder(image, dy0, dy1, dx0, dx1, cv2.BORDER_REFLECT_101)
 mask = cv2.copyMakeBorder(mask, dy0, dy1, dx0, dx1, cv2.BORDER_REFLECT_101)
 return image, mask
def do_invert_intensity(image):
image = np.clip(1-image,0,1)
 return image
def do_brightness_shift(image, alpha=0.125):
image = image + alpha
image = np.clip(image, 0, 1)
 return image
def do_brightness_multiply(image, alpha=1):
image = alpha*image
image = np.clip(image, 0, 1)
 return image
def do_gamma(image, gamma=1.0):
image = image **(1.0 / gamma)
 image = np.clip(image, 0, 1)
 return image
def do_flip_transpose2(image, mask, type=0):
if type==1:
image = image.transpose(1,0)
 image = cv2.flip(image,1)
 mask = mask.transpose(1,0)
 mask = cv2.flip(mask,1)
 if type==2:
image = cv2.flip(image,-1)
 mask = cv2.flip(mask,-1)
 if type==3:
image = image.transpose(1,0)
 image = cv2.flip(image,0)
 mask = mask.transpose(1,0)
 mask = cv2.flip(mask,0)
 if type==4:
image = cv2.flip(image,1)
 mask = cv2.flip(mask,1)
 if type==5:
image = cv2.flip(image,0)
 mask = cv2.flip(mask,0)
 if type==6:
image = cv2.flip(image,1)
 image = image.transpose(1,0)
 image = cv2.flip(image,1)
 mask = cv2.flip(mask,1)
 mask = mask.transpose(1,0)
 mask = cv2.flip(mask,1)
 if type==7:
image = cv2.flip(image,0)
 image = image.transpose(1,0)
 image = cv2.flip(image,1)
 mask = cv2.flip(mask,0)
 mask = mask.transpose(1,0)
 mask = cv2.flip(mask,1)
 return image, mask
def do_shift_scale_crop(image, mask, x0=0, y0=0, x1=1, y1=1):
height, width = image.shape[:2]
image = image[y0:y1,x0:x1]
mask = mask [y0:y1,x0:x1]
image = cv2.resize(image,dsize=(width,height))
mask = cv2.resize(mask,dsize=(width,height))
mask =(mask>0.5 ).astype(np.float32)
 return image, mask
def do_random_shift_scale_crop_pad2(image, mask, limit=0.10):
H, W = image.shape[:2]
dy = int(H*limit)
 y0 = np.random.randint(0,dy)
 y1 = H-np.random.randint(0,dy)
 dx = int(W*limit)
 x0 = np.random.randint(0,dx)
 x1 = W-np.random.randint(0,dx)
 image, mask = do_shift_scale_crop(image, mask, x0, y0, x1, y1)
 return image, mask
def do_shift_scale_rotate2(image, mask, dx=0, dy=0, scale=1, angle=0):
borderMode=cv2.BORDER_REFLECT_101
height, width = image.shape[:2]
sx = scale
sy = scale
cc = math.cos(angle/180*math.pi)*(sx)
 ss = math.sin(angle/180*math.pi)*(sy)
 rotate_matrix = np.array([ [cc,-ss], [ss,cc] ])
 box0 = np.array([ [0,0], [width,0], [width,height], [0,height], ],np.float32)
 box1 = box0 - np.array([width/2,height/2])
 box1 = np.dot(box1,rotate_matrix.T)+ np.array([width/2+dx,height/2+dy])
 box0 = box0.astype(np.float32)
 box1 = box1.astype(np.float32)
 mat = cv2.getPerspectiveTransform(box0,box1)
 image = cv2.warpPerspective(image, mat,(width,height),flags=cv2.INTER_LINEAR,
borderMode=borderMode,borderValue=(0,0,0,))
mask = cv2.warpPerspective(mask, mat,(width,height),flags=cv2.INTER_NEAREST,
borderMode=borderMode,borderValue=(0,0,0,))
mask =(mask>0.5 ).astype(np.float32)
 return image, mask
def do_elastic_transform2(image, mask, grid=32, distort=0.2):
borderMode=cv2.BORDER_REFLECT_101
height, width = image.shape[:2]
x_step = int(grid)
 xx = np.zeros(width,np.float32)
 prev = 0
for x in range(0, width, x_step):
start = x
end = x + x_step
if end > width:
end = width
cur = width
else:
cur = prev + x_step*(1+random.uniform(-distort,distort))
xx[start:end] = np.linspace(prev,cur,end-start)
 prev=cur
y_step = int(grid)
 yy = np.zeros(height,np.float32)
 prev = 0
for y in range(0, height, y_step):
start = y
end = y + y_step
if end > height:
end = height
cur = height
else:
cur = prev + y_step*(1+random.uniform(-distort,distort))
yy[start:end] = np.linspace(prev,cur,end-start)
 prev=cur
map_x,map_y = np.meshgrid(xx, yy)
 map_x = map_x.astype(np.float32)
 map_y = map_y.astype(np.float32)
 image = cv2.remap(image, map_x, map_y, interpolation=cv2.INTER_LINEAR, borderMode=borderMode,borderValue=(0,0,0,))
mask = cv2.remap(mask, map_x, map_y, interpolation=cv2.INTER_NEAREST, borderMode=borderMode,borderValue=(0,0,0,))
mask =(mask>0.5 ).astype(np.float32)
 return image, mask
def do_horizontal_shear2(image, mask, dx=0):
borderMode=cv2.BORDER_REFLECT_101
height, width = image.shape[:2]
dx = int(dx*width)
 box0 = np.array([ [0,0], [width,0], [width,height], [0,height], ],np.float32)
 box1 = np.array([ [+dx,0], [width+dx,0], [width-dx,height], [-dx,height], ],np.float32)
 box0 = box0.astype(np.float32)
 box1 = box1.astype(np.float32)
 mat = cv2.getPerspectiveTransform(box0,box1)
 image = cv2.warpPerspective(image, mat,(width,height),flags=cv2.INTER_LINEAR,
borderMode=borderMode,borderValue=(0,0,0,))
mask = cv2.warpPerspective(mask, mat,(width,height),flags=cv2.INTER_NEAREST,
borderMode=borderMode,borderValue=(0,0,0,))
mask =(mask>0.5 ).astype(np.float32)
 return image, mask
species = model.predict_classes(X )<create_dataframe>
def train_augment(image,mask,index):
if np.random.rand() < 0.5:
image, mask = do_horizontal_flip2(image, mask)
 pass
if np.random.rand() < 0.5:
c = np.random.choice(3)
 if c==0:
image, mask = do_random_shift_scale_crop_pad2(image, mask, 0.125)
 if c==1:
image, mask = do_elastic_transform2(image, mask, grid=10,
distort=np.random.uniform(0,0.1))
if c==2:
image, mask = do_shift_scale_rotate2(image, mask, dx=0, dy=0, scale=1,
angle=np.random.uniform(0,10))
if np.random.rand() < 0.5:
c = np.random.choice(3)
 if c==0:
image = do_brightness_shift(image,np.random.uniform(-0.05,+0.05))
if c==1:
image = do_brightness_multiply(image,np.random.uniform(1-0.05,1+0.05))
if c==2:
image = do_gamma(image,np.random.uniform(1-0.05,1+0.05))
image, mask = do_center_pad_to_factor2(image, mask, factor=32)
 return image,mask,index
ans = pd.DataFrame(file,columns = ["file"])
 ans = ans.join(pd.DataFrame(species,columns=["species"]))
ans["species"] = ans["species"].apply(lambda x: CATEGORIES[int(x)])
 ans.head(20 )<save_to_csv>
train_df = pd.read_csv(".. /input/train.csv", index_col="id", usecols=[0])
 depths_df = pd.read_csv(".. /input/depths.csv", index_col="id")
 train_df = train_df.join(depths_df)
 test_df = depths_df[~depths_df.index.isin(train_df.index)]
ans.to_csv("answers.csv",index=False )<save_model>
train_df["images"] = [np.array(load_img(".. /input/train/images/{}.png".format(idx), grayscale=True)) / 255 for idx in tqdm_notebook(train_df.index)]
model.save("saved_model" )<import_modules>
train_df["masks"] = [np.array(load_img(".. /input/train/masks/{}.png".format(idx), grayscale=True)) / 255 for idx in tqdm_notebook(train_df.index)]
scale = 70
seed = 7<train_model>
train_df["coverage"] = train_df.masks.map(np.sum)/ pow(img_size_ori, 2 )
path_to_images = '.. /input/plant-seedlings-classification/train/*/*.png'
images = glob(path_to_images)
 trainingset = []
traininglabels = []
num = len(images)
 count = 1
for i in images:
print(str(count)+'/'+str(num),end='\r')
 trainingset.append(cv2.resize(cv2.imread(i),(scale,scale)))
 traininglabels.append(i.split('/')[-2])
 count=count+1
trainingset = np.asarray(trainingset)
 traininglabels = pd.DataFrame(traininglabels )<categorify>
def cov_to_class(val):
for i in range(0, 11):
if val * 10 <= i :
return i
train_df["coverage_class"] = train_df.coverage.map(cov_to_class )
labels = preprocessing.LabelEncoder()
labels.fit(traininglabels[0])
 print('Classes'+str(labels.classes_))
encodedlabels = labels.transform(traininglabels[0])
 clearalllabels = np_utils.to_categorical(encodedlabels)
 classes = clearalllabels.shape[1]
print(str(classes))
traininglabels[0].value_counts().plot(kind='pie' )<split>
ids_train, ids_valid, x_train, x_valid, y_train, y_valid, cov_train, cov_test, depth_train, depth_test = train_test_split(
 train_df.index.values,
np.array(train_df.images.map(upsample ).tolist() ).reshape(-1, img_size_target, img_size_target, 1),
np.array(train_df.masks.map(upsample ).tolist() ).reshape(-1, img_size_target, img_size_target, 1),
train_df.coverage.values,
train_df.z.values,
test_size=0.2, stratify=train_df.coverage_class, random_state=1337 )
new_train = new_train/255
x_train,x_test,y_train,y_test = train_test_split(new_train,clearalllabels,test_size=0.1,random_state=seed,stratify=clearalllabels )<train_model>
def ConvBn2D(x,in_dim,out_dim):
x=Conv2D()
generator = ImageDataGenerator(rotation_range = 180,zoom_range = 0.1,width_shift_range = 0.1,height_shift_range = 0.1,horizontal_flip = True,vertical_flip = True)
 generator.fit(x_train )<choose_model_class>
model = UNet(( img_size_target,img_size_target,1),start_ch=16,depth=5,batchnorm=True )
np.random.seed(seed)
 model = Sequential()
model.add(Conv2D(filters=64, kernel_size=(5, 5), input_shape=(scale, scale, 3), activation='relu'))
model.add(BatchNormalization(axis=3))
model.add(Conv2D(filters=64, kernel_size=(5, 5), activation='relu'))
model.add(MaxPooling2D(( 2, 2)))
 model.add(BatchNormalization(axis=3))
model.add(Dropout(0.1))
model.add(Conv2D(filters=128, kernel_size=(5, 5), activation='relu'))
model.add(BatchNormalization(axis=3))
model.add(Conv2D(filters=128, kernel_size=(5, 5), activation='relu'))
model.add(MaxPooling2D(( 2, 2)))
 model.add(BatchNormalization(axis=3))
model.add(Dropout(0.1))
model.add(Conv2D(filters=256, kernel_size=(5, 5), activation='relu'))
model.add(BatchNormalization(axis=3))
model.add(Conv2D(filters=256, kernel_size=(5, 5), activation='relu'))
model.add(MaxPooling2D(( 2, 2)))
 model.add(BatchNormalization(axis=3))
model.add(Dropout(0.1))
model.add(Flatten())
 model.add(Dense(256, activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.5))
model.add(Dense(256, activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.5))
model.add(Dense(classes, activation='softmax'))
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
 model.summary() 
<choose_model_class>
sgd = SGD(lr=0.01, decay=1e-4, momentum=0.9, nesterov=True)
 model.compile(loss=bce_dice_loss, optimizer="adam", metrics=["accuracy",iouMetric] )
lrr = ReduceLROnPlateau(monitor='val_acc',
patience=3,
verbose=1,
factor=0.4,
min_lr=0.00001)
 filepath="weights.best.hdf5"
checkpoints = ModelCheckpoint(filepath, monitor='val_acc',
verbose=1, save_best_only=True, mode='max')
 filepath="weights.last_auto4.hdf5"
checkpoints_full = ModelCheckpoint(filepath, monitor='val_acc',
verbose=1, save_best_only=False, mode='max')
 callbacks_list = [checkpoints, lrr, checkpoints_full]
dataset = np.load(".. /input/plantrecomodels/Data.npz")
 data = dict(zip(( "x_train","x_test","y_train", "y_test"),(dataset[k] for k in dataset)))
 trainX = data['x_train']
testX = data['x_test']
trainY = data['y_train']
testY = data['y_test']
hist = model.fit_generator(generator.flow(trainX, trainY, batch_size=75),
epochs=25, validation_data=(testX, testY),
steps_per_epoch=trainX.shape[0], callbacks=callbacks_list)
<train_model>
seq = iaa.Sequential([
iaa.Fliplr(0.5),
] )
model.load_weights("./weights.best.hdf5")
 dataset = np.load(".. /input/plantrecomodels/Data.npz")
 data = dict(zip(( "x_train","x_test","y_train", "y_test"),(dataset[k] for k in dataset)))
 x_train = data['x_train']
x_test = data['x_test']
y_train = data['y_train']
y_test = data['y_test']
print(model.evaluate(x_train, y_train))
print(model.evaluate(x_test, y_test))<predict_on_test>
def gen_flow_for_two_inputs(X, y):
genX1 = gen.flow(X,y, batch_size=batch_size)
 while True:
X=genX1.next()
imgs=[]
masks=[]
for index in range(len(X[0])) :
img=X[0][index]
mask=X[0][index]
img,mask,_=train_augment(img,mask,0)
 imgs.append(img)
 masks.append(mask)
 imgs=np.array(imgs)
 masks=np.array(masks)
 yield np.expand_dims(imgs,axis=3),np.expand_dims(masks,axis=3 )
y_pred = model.predict(x_test)
 y_class = np.argmax(y_pred, axis = 1)
 y_check = np.argmax(y_test, axis = 1)
 cmatrix = confusion_matrix(y_check, y_class)
 print(cmatrix )<save_to_csv>
model_checkpoint = ModelCheckpoint("./keras.model", save_best_only=True, verbose=1)
 def step_decay(epoch):
if epoch<=60:
return 0.01
elif epoch<=90:
return 0.001
else:
return 0.0001
reduce_lr = LearningRateScheduler(step_decay)
 epochs = 120
batch_size = 32
gen = ImageDataGenerator()
gen_flow = gen_flow_for_two_inputs(x_train, y_train)
 history = model.fit_generator(gen_flow,
validation_data=[x_valid, y_valid],
epochs=epochs,
steps_per_epoch=len(x_train)/ batch_size,
callbacks=[model_checkpoint, reduce_lr] )
newtestimages=newtestimages/255
prediction = model.predict(newtestimages)
 pred = np.argmax(prediction,axis=1)
 predStr = labels.classes_[pred]
result = {'file':tests,'species':predStr}
result = pd.DataFrame(result)
 result.to_csv("Prediction.csv",index=False )<install_modules>
model.load_weights("./keras.model",{"bce_dice_loss":bce_dice_loss,'iouMetric':iouMetric} )
<install_modules>
preds_valid = model.predict(x_valid ).reshape(-1, img_size_target, img_size_target)
 preds_valid = np.array([downsample(x)for x in preds_valid])
 y_valid_ori = np.array([train_df.loc[idx].masks for idx in ids_valid] )
!pip install -q tensorflow_io
!pip install -q tfp-nightly<set_options>
thresholds = np.linspace(0, 1, 50)
 ious = np.array([iou_metric_batch(y_valid_ori, np.int32(preds_valid > threshold)) for threshold in tqdm_notebook(thresholds)] )
!pip install cloud-tpu-client
print(tf.__version__)
 Client().configure_tpu_version(tf.__version__, restart_type='ifNeeded' )<import_modules>
threshold_best_index = np.argmax(ious[9:-10])+ 9
iou_best = ious[threshold_best_index]
threshold_best = thresholds[threshold_best_index]
print(tf.__version__ )<set_options>
def RLenc(img, order='F', format=True):

 bytes = img.reshape(img.shape[0] * img.shape[1], order=order)
 runs = []
r = 0
pos = 1
for c in bytes:
if(c == 0):
if r != 0:
runs.append(( pos, r))
pos += r
r = 0
pos += 1
else:
r += 1
if r != 0:
runs.append(( pos, r))
pos += r
r = 0
if format:
z = ''
for rr in runs:
z += '{} {} '.format(rr[0], rr[1])
 return z[:-1]
else:
return runs
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
AUTO = tf.data.experimental.AUTOTUNE
BATCH_SIZE = 32 * strategy.num_replicas_in_sync<define_variables>
preds_test = model.predict(x_test )
SEED = 777
os.environ['PYTHONHASHSEED'] = str(SEED)
 np.random.seed(SEED)
 tf.random.set_seed(SEED)
 USE_FOLD=2
N_SPLIT=7
SAMPLE_RATE=32000
NUM_CLASSES = 264
PERIOD = 5<load_from_csv>
pred_dict = {idx: RLenc(np.round(downsample(preds_test[i])> threshold_best)) for i, idx in enumerate(tqdm_notebook(test_df.index.values)) }
<define_variables><EOS> 
sub = pd.DataFrame.from_dict(pred_dict,orient='index')
 sub.index.names = ['id']
sub.columns = ['rle_mask']
sub.to_csv('submission.csv' )