Kaggle competition: HuBMAP - Hacking the Kidney
<SOS> metric: Dice Kaggle data source: hubmap-hacking-the-kidney<predict_on_test>
!mkdir -p /tmp/pip/cache/
!cp.. /input/segmentation/segmentation_models/efficientnet_pytorch-0.6.3.xyz /tmp/pip/cache/efficientnet_pytorch-0.6.3.tar.gz
!cp.. /input/segmentation/segmentation_models/pretrainedmodels-0.7.4.xyz /tmp/pip/cache/pretrainedmodels-0.7.4.tar.gz
!cp.. /input/segmentation/segmentation_models/segmentation-models-pytorch-0.1.2.xyz /tmp/pip/cache/segmentation_models_pytorch-0.1.2.tar.gz
!cp.. /input/segmentation/segmentation_models/timm-0.1.20-py3-none-any.whl /tmp/pip/cache/
!cp.. /input/segmentation/segmentation_models/timm-0.2.1-py3-none-any.whl /tmp/pip/cache/
!pip install --no-index --find-links /tmp/pip/cache/ efficientnet-pytorch
!pip install --no-index --find-links /tmp/pip/cache/ segmentation-models-pytorch
predictions = model.predict_classes(x_test/255.)<prepare_output>
warnings.filterwarnings("ignore")
submission['label'] = predictions<save_to_csv>
sz = 512
reduce = 2
TH = 0.4
DATA = '.. /input/hubmap-kidney-segmentation/test/'
df_sample = pd.read_csv('.. /input/hubmap-kidney-segmentation/sample_submission.csv')
 DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 overlap = 230
DEVICE
submission.to_csv("submission.csv",index=False )<set_options>
mean = np.array([0.65459856,0.48386562,0.69428385])
 std = np.array([0.15167958,0.23584107,0.13146145])
 identity = rasterio.Affine(1, 0, 0, 0, 1, 0)
 def img2tensor(img,dtype:np.dtype=np.float32):
if img.ndim==2 : img = np.expand_dims(img,2)
 img = np.transpose(img,(2,0,1))
return torch.from_numpy(img.astype(dtype, copy=False))
rand_seed = 53
%env PYTHONHASHSEED=0
np.random.seed(rand_seed)
 rn.seed(rand_seed)
 tf.config.threading.set_inter_op_parallelism_threads(1)
 tf.config.threading.set_intra_op_parallelism_threads(1)
 tf.random.set_seed(rand_seed )<load_from_csv>
model = [1,2,3]
[model[0],model[1]]
def read_data(file_name):
file_path = '.. /input/Kannada-MNIST/' + file_name
data_df = pd.read_csv(file_path)
 pixels_df = data_df.drop(columns='label')
 pixels_array = pixels_df.to_numpy(dtype=np.uint8)
 reshaped_pixels_array = pixels_array.reshape(-1, 28, 28, 1)
 labels_array = data_df.label.values
return(reshaped_pixels_array, labels_array )<split>
WINDOW = 1024
DATA_PATH = '.. /input/hubmap-kidney-segmentation/'
p = pathlib.Path(DATA_PATH)
 numk = 0
for i, filename in enumerate(p.glob('test/*.tiff')) :
file = rasterio.open(filename.as_posix())
 slices = make_grid(file.shape, window=WINDOW, min_overlap=overlap)
 preds = np.zeros(file.shape, dtype=np.float16)
 print('staring predict test data:',filename)
 print(file.shape)
 print(slices)
 models = []
model = torch.load(f'.. /input/b7-models/best_0_7.pth')
 models.append(model)
 del model; gc.collect()
model = torch.load(f'.. /input/b7-models/best_4.pth')
 models.append(model)
 del model; gc.collect()
model = torch.load(f'.. /input/b7-models/best_3.pth')
 models.append(model)
 del model; gc.collect()
if file.count != 3:
print('Image file with subdatasets as channels')
 layers = [rasterio.open(subd)for subd in file.subdatasets]
for(x1,x2,y1,y2)in slices:
if file.count == 3:
image = file.read([1,2,3],
window=Window.from_slices(( x1,x2),(y1,y2)))
 image = np.moveaxis(image, 0, -1)
 else:
image = np.zeros(( WINDOW, WINDOW, 3), dtype=np.uint8)
 for fl in range(3):
image[:,:,fl] = layers[fl].read(window=Window.from_slices(( x1,x2),(y1,y2)))
 image = cv2.resize(image,(image.shape[1]//reduce,image.shape[0]//reduce),
interpolation = cv2.INTER_AREA)
 image = img2tensor(( image/255 - mean)/std)
 with torch.no_grad() :
image = image.to(DEVICE)
 score = model_pred([models[0]], image[None,...])
 score2 = model_pred([models[1]], torch.flip(image[None,...], [0, 3]))
score2 = torch.flip(score2[None,None,...], [3, 0])[0][0]
score3 = model_pred([models[2]], torch.flip(image[None,...], [1, 2]))
score3 = torch.flip(score3[None,None,...], [2, 1])[0][0]
score =(score + score2 + score3)/3
score = score.cpu()
score = cv2.resize(np.float32(score),(1024, 1024))
preds[x1:x2,y1:y2] = score*score/(preds[x1:x2,y1:y2] + score)+ preds[x1:x2,y1:y2]*preds[x1:x2,y1:y2]/(preds[x1:x2,y1:y2] + score)
 del score,score2,score3,image
del slices,models; gc.collect()
slices = make_grid(file.shape, window=WINDOW, min_overlap=32)
 models = []
model = torch.load(f'.. /input/0924model/best_0_924.pth')
 models.append(model)
 del model; gc.collect()
model = torch.load(f'.. /input/b5testexternal/b5 -test -external/best_0(5 ).pth')
 models.append(model)
 del model; gc.collect()
model = torch.load(f'.. /input/b5testexternal/b5 -test -external/best_4(1 ).pth')
 models.append(model)
 del model; gc.collect()
model = torch.load(f'.. /input/b5testexternal/b5 -test -external/best_3(1 ).pth')
 models.append(model)
 del model; gc.collect()
model = torch.load(f'.. /input/b5testexternal/b5 -test -external/best_1(1 ).pth')
 models.append(model)
 del model; gc.collect()
if file.count != 3:
print('Image file with subdatasets as channels')
 layers = [rasterio.open(subd)for subd in file.subdatasets]
for(x1,x2,y1,y2)in slices:
if file.count == 3:
image = file.read([1,2,3],
window=Window.from_slices(( x1,x2),(y1,y2)))
 image = np.moveaxis(image, 0, -1)
 else:
image = np.zeros(( WINDOW, WINDOW, 3), dtype=np.uint8)
 for fl in range(3):
image[:,:,fl] = layers[fl].read(window=Window.from_slices(( x1,x2),(y1,y2)))
 image = cv2.resize(image,(image.shape[1]//reduce,image.shape[0]//reduce),
interpolation = cv2.INTER_AREA)
 image = img2tensor(( image/255 - mean)/std)
 with torch.no_grad() :
image = image.to(DEVICE)
 score = model_pred([models[0]], image[None,...])
 score2 = model_pred([models[1], models[2]], torch.flip(image[None,...], [0, 3]))
score2 = torch.flip(score2[None,None,...], [3, 0])[0][0]
score3 = model_pred([models[3], models[4]], torch.flip(image[None,...], [1, 2]))
score3 = torch.flip(score3[None,None,...], [2, 1])[0][0]
score =(score+score2+score3)/3
score = score.cpu()
score = cv2.resize(np.float32(score),(1024, 1024))
preds[x1:x2,y1:y2] =(score + preds[x1:x2,y1:y2])/2
preds =(preds > TH ).astype(np.uint8)
 subm[i] = {'id':filename.stem, 'predicted': rle_numba_encode(preds)}
del preds; gc.collect()
<define_variables><EOS> 
submission = pd.DataFrame.from_dict(subm, orient='index')
 submission.to_csv(f'submission.csv', index=False)
 submission
%reset -f out