Kaggle competition: HuBMAP - Hacking the Kidney
<SOS> metric: Dice Kaggle data source: hubmap-hacking-the-kidney<train_model>
! pip install.. /input/kerasapplications/keras-team-keras-applications-3b180cb -f./ --no-index -q
! pip install.. /input/efficientnet/efficientnet-1.1.0/ -f./ --no-index -q
sys.path.append(".. /input/segmentation-models-pytorch-install")
 !pip install -q --no-deps.. /input/deepflash2-lfs
warnings.filterwarnings("ignore" )
def train_model(gen_train, gen_val):
model = build_model()
cbs = [ReduceLROnPlateau(monitor='loss',
factor=0.5,
patience=1,
min_lr=1e-5,
verbose=0,
skip_mismatch=True),
EarlyStopping(monitor='val_loss',
min_delta=1e-6,
patience=10,
verbose=1,
mode='auto',
restore_best_weights=True)]
model.summary()
history = model.fit_generator(gen_train,
epochs=50,
validation_data=gen_val,
validation_steps=len(gen_val),
shuffle=True,
callbacks=cbs,
verbose=1)
 return model, history<predict_on_test>
def rle_encode_less_memory(img):
pixels = img.T.flatten()
pixels[0] = 0
pixels[-1] = 0
runs = np.where(pixels[1:] != pixels[:-1])[0] + 2
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs)
 def load_model_weights(model, file, strict=True):
state = torch.load(file, map_location='cpu')
 stats = state['stats']
model_state = state['model']
model.load_state_dict(model_state, strict=strict)
 return model, stats
def _get_gaussian(patch_size, sigma_scale=1./ 8)-> np.ndarray:
tmp = np.zeros(patch_size)
 center_coords = [i // 2 for i in patch_size]
sigmas = [i * sigma_scale for i in patch_size]
tmp[tuple(center_coords)] = 1
gaussian_importance_map = gaussian_filter(tmp, sigmas, 0, mode='constant', cval=0)
 gaussian_importance_map = gaussian_importance_map / np.max(gaussian_importance_map)* 1
gaussian_importance_map = gaussian_importance_map.astype(np.float32)
 gaussian_importance_map[gaussian_importance_map == 0] = np.min(
 gaussian_importance_map[gaussian_importance_map != 0])
 return gaussian_importance_map
train_gen, val_gen, test_gen = get_generators(data_train, data_test)
 model, history = train_model(train_gen, val_gen)
 y_pred = model.predict_generator(test_gen, use_multiprocessing=True )<train_model>
class HubmapDataset(Dataset):
'HubmapDataset class that does not load the full tiff files.'
def __init__(self, file, stats, scale=3, shift=.8, output_shape=(512,512), s_th = 40):
self.mean, self.std = stats
self.scale = scale
self.shift = shift
self.output_shape = output_shape
self.input_shape = tuple(int(t*scale)for t in self.output_shape)
 self.s_th = s_th
self.p_th = 1000*(self.output_shape[0]//256)**2
identity = rasterio.Affine(1, 0, 0, 0, 1, 0)
 self.data = rasterio.open(file, transform = identity, num_threads='all_cpus')
 if self.data.count != 3:
subdatasets = self.data.subdatasets
self.layers = []
if len(subdatasets)> 0:
for i, subdataset in enumerate(subdatasets, 0):
self.layers.append(rasterio.open(subdataset))
self.slices = []
self.out_slices = []
self.out_data_shape = tuple(int(x//self.scale)for x in self.data.shape)
 start_points = [o//2 for o in self.output_shape]
end_points = [(s - st)for s, st in zip(self.out_data_shape, start_points)]
n_points = [int(s//(o*self.shift)) +1 for s, o in zip(self.out_data_shape, self.output_shape)]
center_points = [np.linspace(st, e, num=n, endpoint=True, dtype=np.int64)for st, e, n in zip(start_points, end_points, n_points)]
for cx in center_points[1]:
for cy in center_points[0]:
slices = tuple(slice(int(( c*self.scale - o/2 ).clip(0, s)) , int(( c*self.scale + o/2 ).clip(max=s)))
 for(c, o, s)in zip(( cy, cx), self.input_shape, self.data.shape))
self.slices.append(slices)
 out_slices = tuple(slice(int(( c - o/2 ).clip(0, s)) , int(( c + o/2 ).clip(max=s)))
 for(c, o, s)in zip(( cy, cx), self.output_shape, self.out_data_shape))
self.out_slices.append(out_slices)
 def __len__(self):
return len(self.slices)
 def __getitem__(self, idx):
if torch.is_tensor(idx):
idx = idx.tolist()
slices = self.slices[idx]
if self.data.count == 3:
img = self.data.read([1, 2, 3],
window=Window.from_slices(*slices)
)
 img = np.moveaxis(img, 0, -1)
 else:
img = np.zeros(( *self.input_shape, 3), dtype=np.uint8)
 for fl in range(3):
img[:, :, fl] = self.layers[fl].read(
 window=Window.from_slices(*slices)
)
 if self.scale!=1:
img = cv2.resize(img, self.output_shape, interpolation = cv2.INTER_AREA)
 hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
 h,s,v = cv2.split(hsv)
 if(s>self.s_th ).sum() <= self.p_th or img.sum() <= self.p_th:
idx = -1
img =(img/255.0 - self.mean)/self.std
img = img.transpose(2, 0, 1 ).astype('float32')
 return torch.from_numpy(img), idx
class Model_pred:
'Class for prediction with multiple models'
def __init__(self, models, use_tta=True, batch_size=32):
self.models = models
self.bs = batch_size
self.tfms = [tta.HorizontalFlip() ] if use_tta else []
def predict(self, ds):
dl = DataLoader(ds, self.bs, num_workers=0, shuffle=False, pin_memory=True)
 pred = np.zeros(ds.out_data_shape, dtype='float32')
 merge_map = np.zeros(ds.out_data_shape, dtype='float32')
 gw_numpy = _get_gaussian(ds.output_shape)
 gw = torch.from_numpy(gw_numpy ).to(device)
 with torch.no_grad() :
for images, idxs in tqdm(iter(dl), total=len(dl)) :
if(( idxs>=0 ).sum() > 0):
images = images[idxs>=0].to(device)
 idxs = idxs[idxs>=0]
merger = tta.Merger()
for t in tta.Compose(self.tfms):
aug_images = t.augment_image(images)
 model_merger = tta.Merger()
for model in tqdm(self.models):
out = model(aug_images)
 out = F.softmax(out, dim=1)
 model_merger.append(out)
 torch.cuda.empty_cache()
out = t.deaugment_mask(model_merger.result())
 merger.append(out)
 batch_smx = merger.result() *gw.view(1,1,*gw.shape)
 batch_smx = [x for x in batch_smx.permute(0,2,3,1 ).cpu().numpy() ]
for smx, idx in zip(batch_smx, idxs):
slcs = ds.out_slices[idx]
pred[slcs] += smx[...,1]
merge_map[slcs] += gw_numpy
pred /= merge_map
return pred
def print_results(history):
print("ACCURACY :")
 print(f"Training accuracy : {history.history['accuracy'][-1]}")
 print(f"Validation accuracy : {history.history['val_accuracy'][-1]}")
 print("
LOSS :")
 print(f"Training categorical crossentropy loss : {history.history['loss'][-1]}")
 print(f"Validation categorical crossentropy loss : {history.history['val_loss'][-1]}" )<prepare_output>
class CONFIG() :
data_path = Path('.. /input/hubmap-kidney-segmentation')
 model_path = Path('.. /input/hubmap-single-fold-models-b1b5')
 scale = 3
shift = 0.8
tile_shape =(512, 512)
 encoder_name = "efficientnet-b4"
encoder_weights = None
in_channels = 3
classes = 2
batch_size = 4
tta = True
threshold = 0.30
cfg = CONFIG()
data_test_label = data_test.copy()
y_pred_label = np.argmax(y_pred, axis=1)
 data_test_label['label'] = y_pred_label<create_dataframe>
THRESHOLD = 0.3
VOTERS = 0.5
WINDOW = 1024
MIN_OVERLAP = 300
NEW_SIZE = 256
SUM_PRED = 128
CHECK=False
def create_submission(data):
df = pd.DataFrame()
df['id'] = data['id']
df['label'] = data['label']
return df<save_to_csv>
def make_grid_tf(shape, window=1024, min_overlap=32):

 x, y = shape
nx = x //(window - min_overlap)+ 1
x1 = np.linspace(0, x, num=nx, endpoint=False, dtype=np.int64)
 x1[-1] = x - window
x2 =(x1 + window ).clip(0, x)
 ny = y //(window - min_overlap)+ 1
y1 = np.linspace(0, y, num=ny, endpoint=False, dtype=np.int64)
 y1[-1] = y - window
y2 =(y1 + window ).clip(0, y)
 slices = np.zeros(( nx,ny, 4), dtype=np.int64)
 for i in range(nx):
for j in range(ny):
slices[i,j] = x1[i], x2[i], y1[j], y2[j]
return slices.reshape(nx*ny,4 )
submission = create_submission(data_test_label)
 submission.to_csv('submission.csv', index=False )<set_options>
models_path_list = [
'.. /input/hubmap-efficientnet-b4/model-fold-4.h5',
'.. /input/hubmap-efficientnet-b4/model-fold-0.h5',
'.. /input/hubmap-efficientnet-b4/model-fold-3.h5',
'.. /input/hubmap-efficientnet-b6-pseudo/model-fold-2.h5',
'.. /input/hubmap-efficientnet-b6-pseudo/model-fold-1.h5',
]
fold_models = []
for fold_model_path in models_path_list:
fold_models.append(tf.keras.models.load_model(fold_model_path,compile = False))
print(len(fold_models))
%matplotlib inline 
<load_from_csv>
deepflash_models=[]

 model = smp.Unet(encoder_name='efficientnet-b6',
encoder_weights=cfg.encoder_weights,
in_channels=cfg.in_channels,
classes=cfg.classes)
 model, stats = load_model_weights(model, '.. /input/hubmap-deepflash-efficientnetb6/'+f'unet_efficientnet-b6.pth')
 if torch.cuda.is_available() : model.cuda()
deepflash_models.append(model)
 del model
gc.collect()
train_data = pd.read_csv('/kaggle/input/Kannada-MNIST/train.csv')
 print(train_data.shape )<load_from_csv>
p = pathlib.Path('.. /input/hubmap-kidney-segmentation')
 ids=[]
for index, filename in tqdm(enumerate(p.glob('test/*.tiff')) ,
total = len(list(p.glob('test/*.tiff')))) :
ids.append(filename.stem)
 df_sample=pd.DataFrame()
df_sample['id']=ids
df_sample['predicted']=np.nan
df_sample = df_sample.set_index('id')
 if df_sample.shape[0]==5:
df_sample = df_sample.iloc[1:2, :]
else:
df_sample=df_sample
print(len(deepflash_models))
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu' )
<load_from_csv><EOS> 
df = pd.DataFrame({'id':names,'predicted':predicts})
 df.to_csv('submission.csv',index=False)
 df.head()