Kaggle competition: HuBMAP - Hacking the Kidney
<SOS> metric: Dice Kaggle data source: hubmap-hacking-the-kidney<set_options>
!pip install.. /input/segmentationmodelspytorch-013/pretrainedmodels-0.7.4-py3-none-any.whl
!pip install.. /input/segmentationmodelspytorch-013/efficientnet_pytorch-0.6.3-py2.py3-none-any.whl
!pip install.. /input/segmentationmodelspytorch-013/timm-0.3.2-py3-none-any.whl
!pip install.. /input/segmentationmodelspytorch-013/segmentation_models_pytorch-0.1.3-py3-none-any.whl
clear_output()
del net
gc.collect()<compute_train_metric>
sample_submission = pd.read_csv('.. /input/hubmap-kidney-segmentation/sample_submission.csv')
 sample_submission = sample_submission.set_index('id')
 seed = 1015
np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 def rle_encode_less_memory(img):

 pixels = img.T.flatten()
pixels[0] = 0
pixels[-1] = 0
runs = np.where(pixels[1:] != pixels[:-1])[0] + 2
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs)
 test_files = sample_submission.index.tolist()
net = load_net(WEIGHT_FILE2)
 results = []
for images, image_ids in tqdm(data_loader):
predictions = make_tta_predictions(images)
 for i, image in enumerate(images):
boxes, scores, labels = run_wbf(predictions, image_index=i)
 boxes =(boxes ).round().astype(np.int32 ).clip(min=0, max=1023)
 image_id = image_ids[i]
score_threshold = STAGE1_SUPPRESS
indexes = np.where(scores>score_threshold)
 boxes = boxes[indexes]
scores = scores[indexes]
boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
bboxes =(format_prediction_string(boxes, scores))
for bbox in bboxes:
result = {
'image_id': image_id,
'width': 1024,
'height': 1024,
'bbox': bbox,
'source': 'pseudo'
}
results.append(result)
 test_df = pd.DataFrame(results, columns=['image_id', 'width', 'height', 'bbox', 'source'])
 test_df.to_csv('pseudo2.csv', index=False)
 test_df.head()
print(len(test_df))
del net
gc.collect()<set_options>
PATH = ".. /input/hubmap-models2"
model_list = ['1_unet-se_resnet50_0.9526_epoch_28.pth', '2_unet-se_resnet50_0.9494_epoch_28.pth', '1_unet-timm-effb0_0.9495_epoch_39.pth', '2_unet-timm-effb0_0.9477_epoch_35.pth', '1_unet-timm-resnest26d_0.9522_epoch_28.pth', '1_unet-se_resnet50_pesudo_0.9572_epoch_26.pth', '1_unet-timm-resnest26d-pesudo_0.9563_epoch_39.pth']
SEED = 42
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
seed_everything(SEED )<load_from_csv>
model_path = list(map(lambda x: os.path.join(PATH, x), model_list))
test_df = pd.read_csv('pseudo.csv')
 bad_boxes = [3687, 117344,173,113947,52868,2159,2169,121633,121634,147504,118211,52727,147552]
marking = pd.concat([pd.read_csv('.. /input/global-wheat-detection/train.csv'),test_df])
 if len(glob(f'.. /input/global-wheat-detection/test/*.jpg')) != 10:
marking.drop(bad_boxes)
 bboxs = np.stack(marking['bbox'].apply(lambda x: np.fromstring(x[1:-1], sep=',')))
 for i, column in enumerate(['x', 'y', 'w', 'h']):
marking[column] = bboxs[:,i]
marking.drop(columns=['bbox'], inplace=True)
 targets = set(marking.source.values)
 if len(glob(f'.. /input/global-wheat-detection/test/*.jpg')) != 10:
targets.remove('arvalis_3')
 targets.remove('usask_1')
 skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
 df_folds = marking[['image_id']].copy()
df_folds.loc[:, 'bbox_count'] = 1
df_folds = df_folds.groupby('image_id' ).count()
df_folds.loc[:, 'source'] = marking[['image_id', 'source']].groupby('image_id' ).min() ['source']
df_folds.loc[:, 'stratify_group'] = np.char.add(
 df_folds['source'].values.astype(str),
df_folds['bbox_count'].apply(lambda x: f'_{x // 15}' ).values.astype(str)
)
 df_folds.loc[:, 'fold'] = 0
print(set(df_folds.source.values))
for fold_number,(train_index, val_index)in enumerate(skf.split(X=df_folds.index, y=df_folds['stratify_group'])) :
df_folds.loc[df_folds.iloc[val_index].index, 'fold'] = fold_number
df_folds = pd.concat([ df_folds.loc[df_folds['source'] == i, :] for i in targets])
 df_folds.loc[df_folds['source']=='pseudo','fold'] = 1
df_folds.loc[df_folds['fold']==4,'fold'] = 1<define_search_model>
models = []
for path in model_path:
model = torch.load(path, map_location= 'cuda')
 model.float()
model.eval()
model.to('cuda')
 models.append(model)
 del model
<define_variables><EOS> 
sz = 512
test_path = '.. /input/hubmap-kidney-segmentation/test/'
for step, person_idx in enumerate(test_files):
print(f'load {step+1}/{len(test_files)} data...')
 img = tiff.imread(test_path + person_idx + '.tiff' ).squeeze()
if img.shape[0] == 3:
img = img.transpose(1,2,0)
 predict_mask_l1 = np.zeros(( img.shape[0], img.shape[1]), dtype = bool)
 landscape =img.shape[0]// 512
portrait = img.shape[1]// 512
sz = 512
print('predict mask...')
 for x in tqdm(range(landscape)) :
for y in range(portrait):
start_x =(512)* x
end_x =(1024)+ start_x
start_y =(512)* y
end_y =(1024)+ start_y
if x == landscape-1:
start_x = img.shape[0] - 1024
end_x = img.shape[0]
if y == portrait-1:
start_y = img.shape[1] - 1024
end_y = img.shape[1]
sample_img = img[start_x : end_x, start_y : end_y,:]
sample_img = cv2.resize(sample_img,(sz,sz),interpolation = cv2.INTER_AREA)/256
sample_img = torch.cuda.FloatTensor(sample_img.transpose([2,0,1])[np.newaxis,...])
 with torch.no_grad() :
sample_pred = models[0].predict(sample_img ).cpu().numpy() [0,0,:,:]
sample_pred = cv2.resize(sample_pred,(1024,1024),interpolation = cv2.INTER_NEAREST)
 sample_pred = np.where(sample_pred > 0.1, True, False ).astype(bool)
 predict_mask_l1[start_x + 256 : end_x - 256, start_y + 256 : end_y - 256] = sample_pred[256:256 + 512,256:256 + 512]
del sample_img
del sample_pred
gc.collect()
predict_mask_l1 = predict_mask_l1.astype(np.uint8)
 contours, hierarchy = cv2.findContours(predict_mask_l1, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)
 predict_mask_l2 = np.zeros(( img.shape[0], img.shape[1]), dtype = bool)
 for cont in tqdm(contours):
center_y, center_x = cont.mean(axis = 0 ).round(0 ).astype(int)[0]
left_x = int(center_x - 512)
 top_y = int(center_y - 512)
 if left_x < 0:
left_x = 0
elif left_x + 1024 > img.shape[0]:
left_x = img.shape[0] - 1024
if top_y < 0:
top_y = 0
elif top_y + 1024 > img.shape[1]:
top_y = img.shape[1] - 1024
sample_img_l2 = img[left_x : left_x + 1024, top_y : top_y+ 1024,:]
sample_img_l2 = cv2.resize(sample_img_l2,(sz,sz),interpolation = cv2.INTER_AREA)/256
sample_img_l2 = torch.cuda.FloatTensor(sample_img_l2.transpose([2,0,1])[np.newaxis,...])
 sample_pred_l2 = None
with torch.no_grad() :
for model in models:
pred_l2 = model.predict(sample_img_l2 ).cpu().numpy() [0,0,:,:]
if sample_pred_l2 is None: sample_pred_l2 = pred_l2
sample_pred_l2 += pred_l2
sample_pred_l2 /= len(models)
 sample_pred_l2 = cv2.resize(sample_pred_l2,(1024,1024),interpolation = cv2.INTER_NEAREST)
 sample_pred_l2 = np.where(sample_pred_l2 > 0.3, True, False ).astype(np.uint8)
 contours_l2, hierarchy = cv2.findContours(sample_pred_l2,
cv2.RETR_EXTERNAL,
cv2.CHAIN_APPROX_NONE)
 if len(contours_l2)< 1:
print('no cotour')
 continue
for cont_l2 in contours_l2:
area = cv2.contourArea(cont_l2)
 min_y, min_x = cont_l2.min(axis = 0 ).round(0 ).astype(int)[0]
max_y, max_x = cont_l2.max(axis = 0 ).round(0 ).astype(int)[0]
if(min_x < 512)and(max_x > 512):
if(min_y < 512)and(max_y > 512):
sample_mask_l2 = np.zeros(sample_pred_l2.shape,
dtype = np.uint8)
 sample_center = cv2.drawContours(sample_mask_l2,
[cont_l2],
0, 
(255, 255, 255),
-1)
 predict_mask_l2[left_x : left_x + 1024,
top_y : top_y+ 1024] =\
np.logical_or(predict_mask_l2[left_x : left_x + 1024,
top_y : top_y+ 1024],
sample_center)
 del predict_mask_l1
del img
gc.collect()
print('convert mask to rle
')
 predict_rle = rle_encode_less_memory(predict_mask_l2)
 sample_submission.loc[person_idx,'predicted'] = predict_rle
del predict_rle
del predict_mask_l2
gc.collect()
sample_submission.reset_index().to_csv('/kaggle/working/submission.csv',index=False)
 sample_submission.reset_index().to_csv('submission.csv', index = False)
 sample_submission = sample_submission.reset_index()
sample_submission.to_csv('/kaggle/working/submission.csv',index=False)
 sample_submission.to_csv('submission.csv', index = False)
 sample_submission