Kaggle competition: HuBMAP - Hacking the Kidney
<SOS> metric: Dice Kaggle data source: hubmap-hacking-the-kidney<choose_model_class>
mod_path = '/kaggle/input/04-04/'
THRESHOLD = 0.4
WINDOW = 1024
MIN_OVERLAP = 300
NEW_SIZE = 512
SUBMISSION_MODE = 'FULL'
CHECKSUM = False
class CNNUnit(nn.Module):
def __init__(self, in_channels, out_channels):
super(CNNUnit, self ).__init__()
self.conv = nn.Conv2d(in_channels=in_channels,
out_channels=out_channels,
kernel_size = 3, stride = 1, padding = 1)
 self.bn = nn.BatchNorm2d(num_features=out_channels, momentum=0.9)
 self.LR = nn.LeakyReLU(inplace=True, negative_slope=0.1)
 def forward(self,x):
output = self.LR(self.bn(self.conv(x)))
 return output<choose_model_class>
import numpy as np
import pandas as pd
import os
import glob
import gc
import rasterio
from rasterio.windows import Window
import pathlib
from tqdm.notebook import tqdm
import cv2
import tensorflow as tf
class FCUnit(nn.Module):
def __init__(self, input_dim, output_dim, initrange):
super(FCUnit, self ).__init__()
self.fc = nn.Linear(in_features=input_dim,
out_features = output_dim)
 nn.init.uniform_(a = -initrange, b = initrange, tensor=self.fc.weight)
 nn.init.zeros_(self.fc.bias)
 self.LR = nn.LeakyReLU(inplace=True)
 self.bn = nn.BatchNorm1d(num_features=output_dim)
 def forward(self, x):
output = self.bn(self.LR(self.fc(x)))
 return output<choose_model_class>
!pip install -U.. /input/kerasapplications/Keras_Applications-1.0.8-py3-none-any.whl
!pip install.. /input/qubvel/efficientnet-1.0.0-py3-none-any.whl
!pip install.. /input/qubvel/image_classifiers-1.0.0-py3-none-any.whl
!pip install.. /input/qubvel-segmentation-model-keras-v101/segmentation_models-master
%env SM_FRAMEWORK=tf.keras
class CNN(nn.Module):
def __init__(self):
super(CNN,self ).__init__()
self.unit1 = CNNUnit(in_channels=1, out_channels=32)
 self.unit2 = CNNUnit(in_channels=32, out_channels=32)
 self.unit3 = CNNUnit(in_channels=32, out_channels=32)
 self.pool1 = nn.MaxPool2d(kernel_size=2, stride = 2)
 self.unit4 = CNNUnit(in_channels=32, out_channels=64)
 self.unit5 = CNNUnit(in_channels=64, out_channels=64)
 self.unit6 = CNNUnit(in_channels=64, out_channels=64)
 self.unit7 = CNNUnit(in_channels=64, out_channels=64)
 self.pool2 = nn.MaxPool2d(kernel_size=2, stride = 2)
 self.unit8 = CNNUnit(in_channels=64, out_channels=128)
 self.unit9 = CNNUnit(in_channels=128, out_channels=128)
 self.pool3 = nn.MaxPool2d(kernel_size=2, stride = 2, padding = 1)
 self.unit10 = CNNUnit(in_channels=128, out_channels=256)
 self.avg = nn.AvgPool2d(kernel_size=2, stride = 2)
 self.fc1 = FCUnit(input_dim=2*2*256, output_dim=64, initrange=0.5)
 self.fc2 = nn.Linear(64,10)
 self.net = nn.Sequential(self.unit1,self.unit2,self.unit3,self.pool1,
self.unit4,self.unit5,self.unit6,self.unit7,self.pool2,
self.unit8,self.unit9,self.pool3,
self.unit10,self.avg)
 self.localization = nn.Sequential(
 nn.Conv2d(1, 8, kernel_size=7),
nn.MaxPool2d(2, stride=2),
nn.ReLU(True),
nn.Conv2d(8, 10, kernel_size=5),
nn.MaxPool2d(2, stride=2),
nn.ReLU(True))
self.fc_loc = nn.Sequential(
 nn.Linear(10 * 3 * 3, 32),
nn.ReLU(True),
nn.Linear(32, 3 * 2))
self.fc_loc[2].weight.data.zero_()
self.fc_loc[2].bias.data.copy_(torch.tensor([1, 0, 0, 0, 1, 0], dtype=torch.float))
def stn(self, x):
xs = self.localization(x)
 xs = xs.view(-1, 10 * 3 * 3)
 theta = self.fc_loc(xs)
 theta = theta.view(-1, 2, 3)
 grid = F.affine_grid(theta, x.size())
 x = F.grid_sample(x, grid)
 return x
def forward(self, image):
output = self.stn(image)
 output = self.net(output)
 output = output.view(output.size(0),-1)
 output = F.dropout(( self.fc1(output)) , p = 1-KEPT_PROB, training = False)
 output = F.log_softmax(self.fc2(output))
return output<categorify>
def rle_encode_less_memory(img):
pixels = img.T.flatten()
pixels[0] = 0
pixels[-1] = 0
runs = np.where(pixels[1:] != pixels[:-1])[0] + 2
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs)
 def make_grid(shape, window=256, min_overlap=32):

 x, y = shape
nx = x //(window - min_overlap)+ 1
x1 = np.linspace(0, x, num=nx, endpoint=False, dtype=np.int64)
 x1[-1] = x - window
x2 =(x1 + window ).clip(0, x)
 ny = y //(window - min_overlap)+ 1
y1 = np.linspace(0, y, num=ny, endpoint=False, dtype=np.int64)
 y1[-1] = y - window
y2 =(y1 + window ).clip(0, y)
 slices = np.zeros(( nx,ny, 4), dtype=np.int64)
 for i in range(nx):
for j in range(ny):
slices[i,j] = x1[i], x2[i], y1[j], y2[j]
return slices.reshape(nx*ny,4 )
class oversampdata(Dataset):
def __init__(self, data):
self.data = torch.FloatTensor(data.values.astype('float'))
def __len__(self):
return len(self.data)
 def __getitem__(self, index):
target = self.data[index][-1]
data_val = self.data[index][:-1]
return data_val, target<train_model>
identity = rasterio.Affine(1, 0, 0, 0, 1, 0)
 fold_models = []
for fold_model_path in glob.glob(mod_path+'*.h5'):
model=sm.Unet('efficientnetb4', encoder_weights=None,activation="sigmoid")
 model.load_weights(fold_model_path)
 fold_models.append(model)
 print(len(fold_models))
def train_func(sub_train_):
train_loss = []
train_acc = []
data = DataLoader(sub_train_, batch_size=BATCH_SIZE, shuffle=True)
 for i,(image, label)in enumerate(data):
optimizer.zero_grad()
image, label = image.to(device), label.to(device)
 image = image.view(-1,1,28,28)
 label = label.long()
output = model(image)
 loss = criterion(output, label)
 train_loss.append(float(loss.item()))
loss.backward()
optimizer.step()
del(image)
 output = output.data.max(dim=1)[1]
train_acc.append(((output.data == label.data ).float().mean() ).item())
 del(label)
 torch.cuda.empty_cache()
if i % 200 == 0:
print("Batch = {},\t loss = {:2.4f},\t accuracy = {}%".format(i, train_loss[-1], train_acc[-1]*100))
def test(data_):
loss = 0
acc = 0
data = DataLoader(data_, batch_size=BATCH_SIZE)
 for image, label in data:
image, label = image.to(device), label.to(device)
 image = image.view(-1,1,28,28)
 label = label.long()
with torch.no_grad() :
output = model(image)
 loss = criterion(output, label)
 loss += float(loss.item())
 del(image)
 output = output.data.max(dim=1)[1]
acc += float(( output.data == label.data ).sum().item() / len(data_))
del(label)
 torch.cuda.empty_cache()
return loss / len(data_), acc<define_variables>
AUTO = tf.data.experimental.AUTOTUNE
image_feature = {
'image': tf.io.FixedLenFeature([], tf.string),
'x1': tf.io.FixedLenFeature([], tf.int64),
'y1': tf.io.FixedLenFeature([], tf.int64)
 }
def _parse_image(example_proto):
example = tf.io.parse_single_example(example_proto, image_feature)
 image = tf.reshape(tf.io.decode_raw(example['image'],out_type=np.dtype('uint8')) ,(P['DIM'],P['DIM'], 3))
return image, example['x1'], example['y1']
def load_dataset(filenames, ordered=True):
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames)
 dataset = dataset.with_options(ignore_order)
 dataset = dataset.map(_parse_image)
 return dataset
def get_dataset(FILENAME):
dataset = load_dataset(FILENAME)
 dataset = dataset.batch(64)
 dataset = dataset.prefetch(AUTO)
 return dataset
BATCH_SIZE = 32
KEPT_PROB = 0.5
N_EPOCHS = 10
LEARNING_RATE = 0.0001<choose_model_class>
p = pathlib.Path('.. /input/hubmap-kidney-segmentation')
 subm = {}
for i, filename in tqdm(enumerate(p.glob('test/*.tiff')) ,
total = len(list(p.glob('test/*.tiff')))) :
print(f'{i+1} Predicting {filename.stem}')
 dataset = rasterio.open(filename.as_posix() , transform = identity)
 preds = np.zeros(dataset.shape, dtype=np.uint8)
 if SUBMISSION_MODE == 'PUBLIC_TFREC' and MIN_OVERLAP == 300 and WINDOW == 1024 and NEW_SIZE == 512:
print('SUBMISSION_MODE: PUBLIC_TFREC')
 fnames = glob.glob('/kaggle/input/hubmap-tfrecords-1024-512-test/test/'+filename.stem+'*.tfrec')
 if len(fnames)>0:
for FILENAME in fnames:
pred = None
for fold_model in fold_models:
tmp = fold_model.predict(get_dataset(FILENAME)) /len(fold_models)
 if pred is None:
pred = tmp
else:
pred += tmp
del tmp
gc.collect()
pred = tf.cast(( tf.image.resize(pred,(WINDOW,WINDOW)) > THRESHOLD),tf.bool ).numpy().squeeze()
idx = 0
for img, X1, Y1 in get_dataset(FILENAME):
for fi in range(X1.shape[0]):
x1 = X1[fi].numpy()
y1 = Y1[fi].numpy()
preds[x1:(x1+WINDOW),y1:(y1+WINDOW)] += pred[idx]
idx += 1
else:
pass
else:
print('SUBMISSION_MODE: FULL')
 slices = make_grid(dataset.shape, window=WINDOW, min_overlap=MIN_OVERLAP)
 if dataset.count != 3:
print('Image file with subdatasets as channels')
 layers = [rasterio.open(subd)for subd in dataset.subdatasets]
for(x1,x2,y1,y2)in slices:
if dataset.count == 3:
image = dataset.read([1,2,3],
window=Window.from_slices(( x1,x2),(y1,y2)))
 image = np.moveaxis(image, 0, -1)
 else:
image = np.zeros(( WINDOW, WINDOW, 3), dtype=np.uint8)
 for fl in range(3):
image[:,:,fl] = layers[fl].read(window=Window.from_slices(( x1,x2),(y1,y2)))
 image = cv2.resize(image,(NEW_SIZE, NEW_SIZE),interpolation = cv2.INTER_AREA)
 image = np.expand_dims(image, 0)
 pred = None
for fold_model in fold_models:
if pred is None:
pred = np.squeeze(fold_model.predict(image))
else:
pred += np.squeeze(fold_model.predict(image))
pred = pred/len(fold_models)
 pred = cv2.resize(pred,(WINDOW, WINDOW))
preds[x1:x2,y1:y2] +=(pred > THRESHOLD ).astype(np.uint8)
 preds =(preds > 0.5 ).astype(np.uint8)
 subm[i] = {'id':filename.stem, 'predicted': rle_encode_less_memory(preds)}
if CHECKSUM:
print('Checksum: '+ str(np.sum(preds)))
 del preds
gc.collect() ;
<split><EOS> 
submission = pd.DataFrame.from_dict(subm, orient='index')
 submission.to_csv('submission.csv', index=False)
 submission.head()