Kaggle competition: HuBMAP - Hacking the Kidney
<SOS> metric: Dice Kaggle data source: hubmap-hacking-the-kidney<data_type_conversions>
!pip install.. /input/packages/pretrainedmodels-0.7.4-py3-none-any.whl
!pip install.. /input/segmentationmodelspytorch/segmentation_models/timm-0.1.20-py3-none-any.whl
!pip install.. /input/packages/efficientnet_pytorch-0.6.3-py2.py3-none-any.whl
!pip install.. /input/segmentationmodelspytorch/segmentation_models/segmentation_models_pytorch-0.1.2-py3-none-any.whl
clear_output()
X = X.astype('float32')/ 255.0<create_dataframe>
!pip install git+https://github.com/qubvel/segmentation_models.pytorch
y = to_categorical(y, num_classes=L)
 pd.DataFrame(y )<split>
sample_submission = pd.read_csv('.. /input/hubmap-kidney-segmentation/sample_submission.csv')
 sample_submission = sample_submission.set_index('id')
 seed = 1015
np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 def rle_encode_less_memory(img):

 pixels = img.T.flatten()
pixels[0] = 0
pixels[-1] = 0
runs = np.where(pixels[1:] != pixels[:-1])[0] + 2
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs)
 test_files = sample_submission.index.tolist()
( X_train, X_ttest, y_train, y_ttest)= train_test_split(X, y, test_size = 0.2, random_state=42)
 print(X_train.shape)
 print(X_ttest.shape)
 del X<choose_model_class>
smp.__version__
daug = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=12,
zoom_range = 0.3,
width_shift_range=0.2,
height_shift_range=0.2,
horizontal_flip=False,
vertical_flip=False
)<choose_model_class>
model = smp.Unet(
 encoder_name='timm-efficientnet-b7',
encoder_weights='noisy-student',
in_channels = 3,
classes=1,
activation = 'sigmoid'
)
class CustomNet(object):
@staticmethod
def build(width, height, num_classes, depth=3):
model = Sequential()
input_shape =(height, width, depth)
 chan_dim = -1
model.add(Conv2D(64,(3, 3), padding='same', input_shape=input_shape))
model.add(LeakyReLU(alpha=0.1))
model.add(BatchNormalization(axis=chan_dim))
model.add(Conv2D(64,(3, 3), padding='same'))
model.add(LeakyReLU(alpha=0.1))
model.add(BatchNormalization(axis=chan_dim))
model.add(Conv2D(64,(3, 3), padding='same'))
model.add(LeakyReLU(alpha=0.1))
model.add(BatchNormalization(axis=chan_dim))
model.add(Conv2D(96,(3, 3), padding='same'))
model.add(LeakyReLU(alpha=0.1))
model.add(BatchNormalization(axis=chan_dim))
model.add(Conv2D(96,(3, 3), padding='same'))
model.add(LeakyReLU(alpha=0.1))
model.add(BatchNormalization(axis=chan_dim))
model.add(Conv2D(96,(3, 3), padding='same'))
model.add(LeakyReLU(alpha=0.1))
model.add(BatchNormalization(axis=chan_dim))
model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Dropout(0.2))
model.add(Conv2D(128,(3, 3), padding='same'))
model.add(LeakyReLU(alpha=0.1))
model.add(BatchNormalization(axis=chan_dim))
model.add(Conv2D(128,(3, 3), padding='same'))
model.add(LeakyReLU(alpha=0.1))
model.add(BatchNormalization(axis=chan_dim))
model.add(Conv2D(128,(3, 3), padding='same'))
model.add(LeakyReLU(alpha=0.1))
model.add(BatchNormalization(axis=chan_dim))
model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Dropout(0.2))
model.add(Conv2D(256,(3, 3), padding='same'))
model.add(LeakyReLU(alpha=0.1))
model.add(BatchNormalization(axis=chan_dim))
model.add(Conv2D(256,(3, 3), padding='same'))
model.add(LeakyReLU(alpha=0.1))
model.add(BatchNormalization(axis=chan_dim))
model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Dropout(0.2))
model.add(Flatten())
 model.add(Dense(256))
model.add(LeakyReLU(alpha=0.1))
model.add(BatchNormalization())
 model.add(Dense(128))
model.add(LeakyReLU(alpha=0.1))
model.add(BatchNormalization())
 model.add(Dropout(0.5))
model.add(Dense(num_classes))
model.add(Activation('softmax'))
print(model.summary())
 return model<choose_model_class>
PATH = ".. /input/hubmap-models2/"
model_names = [
"1_unet-timm-effb7_0.9509_epoch_28.pth",
"2_unet-timm-effb7_0.9488_epoch_28.pth",
"3_unet-timm-effb7_0.9503_epoch_29.pth",
"4_unet-timm-effb7_0.9500_epoch_28.pth",
"5_unet-timm-effb7_0.9518_epoch_27.pth",
]
models = []
for model_name in model_names:
models.append(torch.load(PATH + model_name, map_location= 'cpu'))
<define_variables><EOS> 
sz = 512
test_path = '.. /input/hubmap-kidney-segmentation/test/'
for step, person_idx in enumerate(test_files):
print(f'load {step+1}/{len(test_files)} data...')
 img = tiff.imread(test_path + person_idx + '.tiff' ).squeeze()
if img.shape[0] == 3:
img = img.transpose(1,2,0)
 predict_mask_l1 = np.zeros(( img.shape[0], img.shape[1]), dtype = bool)
 landscape =img.shape[0]// 512
portrait = img.shape[1]// 512
sz = 512
print('predict mask...')
 for x in tqdm(range(landscape)) :
for y in range(portrait):
start_x =(512)* x
end_x =(1024)+ start_x
start_y =(512)* y
end_y =(1024)+ start_y
if x == landscape-1:
start_x = img.shape[0] - 1024
end_x = img.shape[0]
if y == portrait-1:
start_y = img.shape[1] - 1024
end_y = img.shape[1]
sample_img = img[start_x : end_x, start_y : end_y,:]
sample_img = cv2.resize(sample_img,(sz,sz),interpolation = cv2.INTER_AREA)/256
sample_img = torch.cuda.FloatTensor(sample_img.transpose([2,0,1])[np.newaxis,...])
 sample_pred = np.zeros([512,512])
 for model in models:
sample_pred += model.predict(sample_img ).cpu().numpy() [0,0,:,:]
sample_pred = sample_pred/len(models)
 sample_pred = cv2.resize(sample_pred,(1024,1024),interpolation = cv2.INTER_NEAREST)
 sample_pred = np.where(sample_pred > 0.3, True, False ).astype(bool)
 predict_mask_l1[start_x + 256 : end_x - 256, start_y + 256 : end_y - 256] = sample_pred[256:256 + 512,256:256 + 512]
del sample_img
del sample_pred
gc.collect()
predict_mask_l1 = predict_mask_l1.astype(np.uint8)
 contours, hierarchy = cv2.findContours(predict_mask_l1, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)
 predict_mask_l2 = np.zeros(( img.shape[0], img.shape[1]), dtype = bool)
 for cont in tqdm(contours):
center_y, center_x = cont.mean(axis = 0 ).round(0 ).astype(int)[0]
left_x = int(center_x - 512)
 top_y = int(center_y - 512)
 if left_x < 0:
left_x = 0
elif left_x + 1024 > img.shape[0]:
left_x = img.shape[0] - 1024
if top_y < 0:
top_y = 0
elif top_y + 1024 > img.shape[1]:
top_y = img.shape[1] - 1024
sample_img_l2 = img[left_x : left_x + 1024, top_y : top_y+ 1024,:]
sample_img_l2 = cv2.resize(sample_img_l2,(sz,sz),interpolation = cv2.INTER_AREA)/256
sample_img_l2 = torch.cuda.FloatTensor(sample_img_l2.transpose([2,0,1])[np.newaxis,...])
 sample_pred_l2 = np.zeros([512,512])
 for model in models:
sample_pred_l2 += model.predict(sample_img_l2 ).cpu().numpy() [0,0,:,:]
sample_pred_l2 = sample_pred_l2/len(models)
 sample_pred_l2 = cv2.resize(sample_pred_l2,(1024,1024),interpolation = cv2.INTER_NEAREST)
 sample_pred_l2 = np.where(sample_pred_l2 > 0.5, True, False ).astype(np.uint8)
 contours_l2, hierarchy = cv2.findContours(sample_pred_l2,
cv2.RETR_EXTERNAL,
cv2.CHAIN_APPROX_NONE)
 if len(contours_l2)< 1:
print('no contour')
 continue
for cont_l2 in contours_l2:
min_y, min_x = cont_l2.min(axis = 0 ).round(0 ).astype(int)[0]
max_y, max_x = cont_l2.max(axis = 0 ).round(0 ).astype(int)[0]
if(min_x < 512)and(max_x > 512):
if(min_y < 512)and(max_y > 512):
sample_mask_l2 = np.zeros(sample_pred_l2.shape,
dtype = np.uint8)
 sample_center = cv2.drawContours(sample_mask_l2,
[cont_l2],
0, 
(255, 255, 255),
-1)
 predict_mask_l2[left_x : left_x + 1024,
top_y : top_y+ 1024] =\
np.logical_or(predict_mask_l2[left_x : left_x + 1024,
top_y : top_y+ 1024],
sample_center)
 del predict_mask_l1
del img
gc.collect()
print('convert mask to rle
')
 predict_rle = rle_encode_less_memory(predict_mask_l2)
 sample_submission.loc[person_idx,'predicted'] = predict_rle
del predict_rle
del predict_mask_l2
gc.collect()
sample_submission = sample_submission.reset_index()
sample_submission.to_csv('/kaggle/working/submission.csv',index=False )