Kaggle competition: HuBMAP - Hacking the Kidney
<SOS> metric: Dice Kaggle data source: hubmap-hacking-the-kidney<categorify>
sys.path.append(".. /input/segmentation-models-pytorch-install")
 !pip install -q --no-deps.. /input/deepflash2-lfs
warnings.filterwarnings("ignore" )
sub_train = oversampdata(train_data)
 sub_test = oversampdata(test_data )<train_model>
def rle_encode_less_memory(img):
pixels = img.T.flatten()
pixels[0] = 0
pixels[-1] = 0
runs = np.where(pixels[1:] != pixels[:-1])[0] + 2
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs)
 def load_model_weights(model, file, strict=True):
state = torch.load(file, map_location='cpu')
 stats = state['stats']
model_state = state['model']
model.load_state_dict(model_state, strict=strict)
 return model, stats
def _get_gaussian(patch_size, sigma_scale=1./ 8)-> np.ndarray:
tmp = np.zeros(patch_size)
 center_coords = [i // 2 for i in patch_size]
sigmas = [i * sigma_scale for i in patch_size]
tmp[tuple(center_coords)] = 1
gaussian_importance_map = gaussian_filter(tmp, sigmas, 0, mode='constant', cval=0)
 gaussian_importance_map = gaussian_importance_map / np.max(gaussian_importance_map)* 1
gaussian_importance_map = gaussian_importance_map.astype(np.float32)
 gaussian_importance_map[gaussian_importance_map == 0] = np.min(
 gaussian_importance_map[gaussian_importance_map != 0])
 return gaussian_importance_map
for epoch in range(N_EPOCHS):
start_time = time.time()
train_func(sub_train)
 valid_loss, valid_acc = test(sub_test)
 secs = int(time.time() - start_time)
 mins = secs / 60
secs = secs % 60
print('Epoch: %d' %(epoch + 1), " | time in %d minutes, %d seconds" %(mins, secs))
print('\tLoss: {valid_loss:.4f}(test)\t|\tAcc: {valid_acc:.1f}%(test)\t|'.
 format(valid_loss = valid_loss, valid_acc = valid_acc * 100))<load_from_csv>
class HubmapDataset(Dataset):
'HubmapDataset class that does not load the full tiff files.'
def __init__(self, file, stats, scale=3, shift=.8, output_shape=(512,512), s_th = 40):
self.mean, self.std = stats
self.scale = scale
self.shift = shift
self.output_shape = output_shape
self.input_shape = tuple(int(t*scale)for t in self.output_shape)
 self.s_th = s_th
self.p_th = 1000*(self.output_shape[0]//256)**2
identity = rasterio.Affine(1, 0, 0, 0, 1, 0)
 self.data = rasterio.open(file, transform = identity, num_threads='all_cpus')
 if self.data.count != 3:
subdatasets = self.data.subdatasets
self.layers = []
if len(subdatasets)> 0:
for i, subdataset in enumerate(subdatasets, 0):
self.layers.append(rasterio.open(subdataset))
self.slices = []
self.out_slices = []
self.out_data_shape = tuple(int(x//self.scale)for x in self.data.shape)
 start_points = [o//2 for o in self.output_shape]
end_points = [(s - st)for s, st in zip(self.out_data_shape, start_points)]
n_points = [int(s//(o*self.shift)) +1 for s, o in zip(self.out_data_shape, self.output_shape)]
center_points = [np.linspace(st, e, num=n, endpoint=True, dtype=np.int64)for st, e, n in zip(start_points, end_points, n_points)]
for cx in center_points[1]:
for cy in center_points[0]:
slices = tuple(slice(int(( c*self.scale - o/2 ).clip(0, s)) , int(( c*self.scale + o/2 ).clip(max=s)))
 for(c, o, s)in zip(( cy, cx), self.input_shape, self.data.shape))
self.slices.append(slices)
 out_slices = tuple(slice(int(( c - o/2 ).clip(0, s)) , int(( c + o/2 ).clip(max=s)))
 for(c, o, s)in zip(( cy, cx), self.output_shape, self.out_data_shape))
self.out_slices.append(out_slices)
 def __len__(self):
return len(self.slices)
 def __getitem__(self, idx):
if torch.is_tensor(idx):
idx = idx.tolist()
slices = self.slices[idx]
if self.data.count == 3:
img = self.data.read([1, 2, 3],
window=Window.from_slices(*slices)
)
 img = np.moveaxis(img, 0, -1)
 else:
img = np.zeros(( *self.input_shape, 3), dtype=np.uint8)
 for fl in range(3):
img[:, :, fl] = self.layers[fl].read(
 window=Window.from_slices(*slices)
)
 if self.scale!=1:
img = cv2.resize(img, self.output_shape, interpolation = cv2.INTER_AREA)
 hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
 h,s,v = cv2.split(hsv)
 if(s>self.s_th ).sum() <= self.p_th or img.sum() <= self.p_th:
idx = -1
img =(img/255.0 - self.mean)/self.std
img = img.transpose(2, 0, 1 ).astype('float32')
 return torch.from_numpy(img), idx
class Model_pred:
'Class for prediction with multiple models'
def __init__(self, models, use_tta=True, batch_size=32):
self.models = models
self.bs = batch_size
self.tfms = [tta.HorizontalFlip() , tta.VerticalFlip() ] if use_tta else []
def predict(self, ds):
dl = DataLoader(ds, self.bs, num_workers=0, shuffle=False, pin_memory=True)
 pred = np.zeros(ds.out_data_shape, dtype='float32')
 merge_map = np.zeros(ds.out_data_shape, dtype='float32')
 gw_numpy = _get_gaussian(ds.output_shape)
 gw = torch.from_numpy(gw_numpy ).to(device)
 with torch.no_grad() :
for images, idxs in tqdm(iter(dl), total=len(dl)) :
if(( idxs>=0 ).sum() > 0):
images = images[idxs>=0].to(device)
 idxs = idxs[idxs>=0]
merger = tta.Merger()
for t in tta.Compose(self.tfms):
aug_images = t.augment_image(images)
 model_merger = tta.Merger()
for model in self.models:
out = model(aug_images)
 out = F.softmax(out, dim=1)
 model_merger.append(out)
 out = t.deaugment_mask(model_merger.result())
 merger.append(out)
 batch_smx = merger.result() *gw.view(1,1,*gw.shape)
 batch_smx = [x for x in batch_smx.permute(0,2,3,1 ).cpu().numpy() ]
for smx, idx in zip(batch_smx, idxs):
slcs = ds.out_slices[idx]
pred[slcs] += smx[...,1]
merge_map[slcs] += gw_numpy
pred /= merge_map
return pred
test_data = pd.read_csv('.. /input/Kannada-MNIST/test.csv')
 test_data = test_data.drop(['id'], axis = 1)
 test_data = test_data/255.0
test_data = torch.FloatTensor(test_data.values.astype('float'))
test_data = test_data.view(-1,1,28,28 )<predict_on_test>
class CONFIG() :
data_path = Path('.. /input/hubmap-kidney-segmentation')
 model_path = Path('.. /input/hubmap-models-final/scale_models')
 scale = 3
shift = 0.8
tile_shape =(512, 512)
 encoder_name = "efficientnet-b2"
encoder_weights = None
in_channels = 3
classes = 2
batch_size = 32
tta = True
threshold = 0.5
cfg = CONFIG()
def predict(data):
model.eval()
predictions = []
for image in data:
image = image.view(1,1,28,28)
 image = image.to(device)
 output = model(image)
 del image
predictions.append(int(output.argmax(1)))
 return predictions<predict_on_test>
df_sample = pd.read_csv(cfg.data_path/'sample_submission.csv', index_col='id')
 device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 MODELS = sorted([f for f in cfg.model_path.iterdir() if f.suffix=='.pth'])
 print(f'Found {len(MODELS)} models', *MODELS)
 models = []
for i, m_path in enumerate(MODELS):
model = smp.Unet(encoder_name=cfg.encoder_name,
encoder_weights=cfg.encoder_weights,
in_channels=cfg.in_channels,
classes=cfg.classes)
 model, stats = load_model_weights(model, m_path)
 model.float()
model.eval()
model.to(device)
 models.append(model )
<save_to_csv><EOS> 
df = pd.DataFrame({'id':names,'predicted':preds})
 df.to_csv('submission.csv',index=False)
 df.head()