Kaggle competition: HuBMAP - Hacking the Kidney
<SOS> metric: Dice Kaggle data source: hubmap-hacking-the-kidney<save_to_csv>
!pip install.. /input/segmentation-models-pytorch-0-1-3/pretrainedmodels-0.7.4/pretrainedmodels-0.7.4
!pip install.. /input/segmentation-models-pytorch-0-1-3/efficientnet_pytorch-0.6.3/efficientnet_pytorch-0.6.3
!pip install.. /input/segmentation-models-pytorch-0-1-3/timm-0.3.2-py3-none-any.whl
!pip install.. /input/segmentation-models-pytorch-0-1-3/segmentation_models.pytorch.0.1.3/segmentation_models.pytorch.0.1.3
sample_submission_df.to_csv('submission.csv', index=False)
 print('Done!' )<import_modules>
import os
import gc
import cv2
import pdb
import glob
import pytz
import warnings
import pickle
import random
import numpy as np
import pandas as pd
from tqdm import tqdm_notebook as tqdm
from torch.optim.lr_scheduler import ReduceLROnPlateau, ExponentialLR
from sklearn.model_selection import KFold
import torch
import torch.nn as nn
from torch.nn import functional as F
import torch.backends.cudnn as cudnn
from torch.utils.data import DataLoader, Dataset, sampler
from albumentations.pytorch import ToTensorV2
import segmentation_models_pytorch as smp
import tifffile as tiff
import rasterio
from rasterio.windows import Window
import albumentations as A
from keras.optimizers import SGD
from keras.models import Sequential
from keras.utils import to_categorical
from keras.preprocessing.image import ImageDataGenerator
from keras.layers.normalization import BatchNormalization
from keras.layers import Dense, Dropout, Conv2D, Flatten, MaxPool2D
from keras.callbacks import EarlyStopping, History, LearningRateScheduler<define_variables>
Compose,
CenterCrop,
CLAHE,
Resize,
Normalize
)
data_dir = "/kaggle/input/Kannada-MNIST/"<load_from_csv>
height, width = 1024, 1024
reduce = 2
THRESHOLD = 0.40
window = 2048
min_overlap = 256
DATA = '.. /input/hubmap-kidney-segmentation/test/'
MODELS_EFFNET = [".. /input/effnetb4-20epochs-5folds/FOLD-0-model.pth",
".. /input/effnetb4-20epochs-5folds/FOLD-1-model.pth",
".. /input/effnetb4-20epochs-5folds/FOLD-2-model.pth",
".. /input/effnetb4-20epochs-5folds/FOLD-3-model.pth",
".. /input/effnetb4-20epochs-5folds/FOLD-4-model.pth"
]
MODELS_RESNET = [".. /input/resnet34-5folds-original-data-0927/FOLD-0-model_resnet34.pth",
".. /input/resnet34-5folds-original-data-0927/FOLD-1-model_resnet34.pth",
".. /input/resnet34-5folds-original-data-0927/FOLD-2-model_resnet34.pth",
".. /input/resnet34-5folds-original-data-0927/FOLD-3-model_resnet34.pth",
".. /input/resnet34-5folds-original-data-0927/FOLD-4-model_resnet34.pth",]
df_sample = pd.read_csv('.. /input/hubmap-kidney-segmentation/sample_submission.csv')
 batch_size = 15
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu' )
train_df = pd.read_csv(data_dir+"train.csv")
 train_df.head(2 )<categorify>
def rle_encode_less_memory(img):
pixels = img.T.flatten()
pixels[0] = 0
pixels[-1] = 0
runs = np.where(pixels[1:] != pixels[:-1])[0] + 2
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs )
y = train_df.values[:, 0]
y = to_categorical(y)
 print("Shape of y: ", y.shape)
 print("Sample of y: ", y[0] )<train_model>
class HuBMAPEffNet(nn.Module):
def __init__(self):
super(HuBMAPEffNet, self ).__init__()
self.cnn_model = smp.Unet('efficientnet-b4', classes=1, encoder_weights=None)
 def forward(self, imgs):
img_segs = self.cnn_model(imgs)
 return img_segs
class HuBMAPResNet(nn.Module):
def __init__(self):
super(HuBMAPResNet, self ).__init__()
self.cnn_model = smp.Unet('resnet34', classes=1, encoder_weights=None)
 def forward(self, imgs):
img_segs = self.cnn_model(imgs)
 return img_segs
X = train_df.values[:, 1:]/255.0
X = X.reshape(-1, 28, 28, 1)
 print("Shape of X: ", X.shape)
 print("Sample of X: ")
 plt.imshow(X[0].reshape(28, 28))<split>
modelsEffnet = []
for path in MODELS_EFFNET:
state_dict = torch.load(path, map_location=torch.device('cpu'))
model = HuBMAPEffNet().to(device)
 model.load_state_dict(state_dict)
 model.eval()
model.to(device)
 modelsEffnet.append(model)
 del state_dict
modelsResnet = []
for path in MODELS_RESNET:
state_dict = torch.load(path, map_location=torch.device('cpu'))
model = HuBMAPResNet().to(device)
 model.load_state_dict(state_dict)
 model.eval()
model.to(device)
 modelsResnet.append(model)
 del state_dict
models = modelsEffnet + modelsResnet
print(len(models))
validation_split =.15
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=validation_split, stratify=y, random_state=seed )<choose_model_class>
def Make_prediction(img, tta = True):
pred = None
with torch.no_grad() :
for model in models:
p_tta = None
p = model(img)
 p = torch.sigmoid(p ).detach()
if p_tta is None:
p_tta = p
else:
p_tta += p
if tta:
flips = [[-1],[-2],[-2,-1]]
for f in flips:
imgf = torch.flip(img, f)
 p = model(imgf)
 p = torch.flip(p, f)
 p_tta += torch.sigmoid(p ).detach()
p_tta /=(1+len(flips))
if pred is None:
pred = p_tta
else:
pred += p_tta
pred /= len(models)
 return pred
class CNN_Modeling() :
def __init__(self, model_confs):
self.epochs = 30
self.batch_size = 80
self.annealer = LearningRateScheduler(lambda x: 1e-3 * 0.95 ** x, verbose=0)
 self.models = []
self.history = []
self.model_names = []
self.model_confs = model_confs
def build_models(self, feature_maps, kernel_size, dense_size, drop_rate):
model = Sequential()
for i, fm in enumerate(feature_maps):
model.add(Conv2D(fm, kernel_size=kernel_size, padding='same', activation='relu', input_shape=(28, 28, 1)))
 if i == len(feature_maps)-1:
model.add(MaxPool2D())
 else:
model.add(MaxPool2D(padding='same'))
model.add(Dropout(drop_rate))
model.add(Flatten())
 for dns in dense_size:
if dns>0:
model.add(Dense(dns, activation='relu'))
model.add(Dropout(drop_rate))
model.add(Dense(10, activation='softmax'))
model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])
 return model
def get_model_name(self, conf):
name = "-".join(
 ["{}C{}P1".format(ft, conf['kernel_size'])for ft in conf['feature_maps']])
 name = name + "-" + "-".join(map(str, conf['dense_size']))
name = name + '-D%d'%round(conf["drop_rate"]*100)
 return name
def train_models(self, _x_train, _y_train, _x_val, _y_val):
self.models = [None]*len(self.model_confs)
 self.history = [None]*len(self.model_confs)
 self.model_names = [None]*len(self.model_confs)
 for i, model_conf in enumerate(self.model_confs):
self.models[i] = self.build_models(**model_conf)
 self.model_names[i] = self.get_model_name(model_conf)
 self.history[i] = self.models[i].fit(_x_train,_y_train, batch_size=self.batch_size, epochs=self.epochs,
validation_data =(_x_val,_y_val), callbacks=[self.annealer], verbose=0)
 print("CNN {0}: Epochs={1:d}, Train accuracy={2:.5f}, Validation accuracy={3:.5f}".format(
 self.model_names[i], self.epochs, max(self.history[i].history['accuracy']),
max(self.history[i].history['val_accuracy'])) 
)
 def plot_accuracy_chart(self, accuracy='val_accuracy'):
plt.figure(figsize=(15,5))
for i in range(len(self.models)) :
sns.lineplot(
 x=range(self.epochs),
y=self.history[i].history[accuracy],
label=self.model_names[i] 
)
 def predict(self, model_name, _x_predict):
given_model = self.models[self.model_names.index(model_name)]
return given_model.predict_classes(_x_predict)
 def get_model(self, model_name):
return self.models[self.model_names.index(model_name)]<predict_on_test>
names, predictions = [],[]
for idx, row in tqdm(df_sample.iterrows() ,total=len(df_sample)) :
imageId = row['id']
data = rasterio.open(os.path.join(DATA, imageId+'.tiff'), transform = identity, num_threads='all_cpus')
 preds = np.zeros(data.shape, dtype=np.uint8)
 dataset = HuBMAPDataset(data)
 dataloader = DataLoader(dataset, batch_size, num_workers=0, shuffle=False, pin_memory=True)
 for i,(img, vertices)in enumerate(dataloader):
img = img.to(device)
 pred = Make_prediction(img)
 pred = pred.squeeze().cpu().numpy()
vertices = vertices.numpy()
for p, vert in zip(pred, vertices):
x1, x2, y1, y2 = vert
p = cv2.resize(p,(window, window))
preds[x1:x2,y1:y2] +=(p > THRESHOLD ).astype(np.uint8)
 preds =(preds > 0.5 ).astype(np.uint8)
 rle = rle_encode_less_memory(preds)
 names.append(imageId)
 predictions.append(rle)
 del preds, dataset, dataloader
gc.collect()
val_pred_labels = cnn_model_before_data_aug.predict("32C5P1-64C5P1-128C5P1-256-D30", X_val)
 y_val_labels = np.argmax(y_val, axis=1)
 misclassses = np.where(np.argmax(y_val, axis=1)!= val_pred_labels)[0]
print("Number of misclassifications: ", len(misclassses))<train_model>
print('replacement' )
datagen = ImageDataGenerator(
 rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1 
)
 model_name = "32C5P1-64C5P1-128C5P1-256-D30"
bes_model_so_far = cnn_model_before_data_aug.get_model(model_name)
 epochs = 50
history = bes_model_so_far.fit_generator(datagen.flow(X_train, y_train, batch_size=64),
epochs=epochs, steps_per_epoch=X_train.shape[0]//64,validation_data=(X_val,y_val),
callbacks=[cnn_model_before_data_aug.annealer], verbose=0)
 print("CNN {0}: Epochs={1:d}, Train accuracy={2:.5f}, Validation accuracy={3:.5f}".format(
 model_name,epochs,max(history.history['accuracy']),max(history.history['val_accuracy'])) )<load_from_csv>
df = pd.DataFrame({'id':names,'predicted':predictions})
 df['predicted'].loc[df[df.id == 'd488c759a'].index] = ''
df.to_csv('submission.csv', index=False )