Kaggle competition: Prostate cANcer graDe Assessment (PANDA) Challenge
<SOS> metric: QuadraticWeightedKappa Kaggle data source: prostate-cancer-grade-assessment-(panda)-challenge<concatenate>
!pip install.. /input/spamspip/spams-2.6.1
X_test_input = np.asarray([np.concatenate([X_test[i][3],X_test[i+1][3], X_test[i+2][3]], axis=1)for i in range(0,len(X_test), 3)])
 np.save("X_test.npy",X_test_input)
 X_test_input.shape<load_from_csv>
!pip install.. /input/imagecodecs/imagecodecs-2020.5.30-cp37-cp37m-manylinux2010_x86_64.whl
submission = pd.read_csv('.. /input/sample_submission.csv')
 print(len(submission))
submission.head()<predict_on_test>
import numpy as np
import pandas as pd
import os
preds_test = []
for i in range(N_SPLITS):
model.load_weights('weights_{}.h5'.format(i))
pred = model.predict(X_test_input, batch_size=300, verbose=1)
 pred_3 = []
for pred_scalar in pred:
for i in range(3):
pred_3.append(pred_scalar)
 preds_test.append(pred_3)
<data_type_conversions>
DEBUG = False
preds_test =(np.squeeze(np.mean(preds_test, axis=0)) > best_threshold ).astype(np.int)
 preds_test.shape<save_to_csv>
sys.path = [
'.. /input/efficientnet/EfficientNet-PyTorch-master/',
'.. /input/staintoolszip/',
] + sys.path
submission['target'] = preds_test
submission.to_csv('submission.csv', index=False)
 submission.head()<import_modules>
import skimage.io
import numpy as np
import pandas as pd
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.utils.data import DataLoader, Dataset
from efficientnet_pytorch import model as enet
import matplotlib.pyplot as plt
from tqdm import tqdm_notebook as tqdm
import cv2
import pandas as pd
import pyarrow.parquet as pq
import os
import numpy as np
from keras.layers import *
from keras.models import Model
from tqdm import tqdm
from sklearn.model_selection import train_test_split
from keras import backend as K
from keras import optimizers
from sklearn.model_selection import GridSearchCV, StratifiedKFold,RepeatedStratifiedKFold
from keras.callbacks import *
from sklearn.preprocessing import MinMaxScaler
from numba import jit
from math import log, floor
from sklearn.neighbors import KDTree
from scipy.signal import periodogram, welch
import tensorflow as tf<define_variables>
import staintools
N_SPLITS = 5
sample_size = 800000<compute_test_metric>
data_dir = '.. /input/prostate-cancer-grade-assessment'
df_train = pd.read_csv(os.path.join(data_dir, 'train.csv'))
df_test = pd.read_csv(os.path.join(data_dir, 'test.csv'))
df_sub = pd.read_csv(os.path.join(data_dir, 'sample_submission.csv'))
model_dir = '.. /input/4stains'
image_folder = os.path.join(data_dir, 'test_images')
 is_test = os.path.exists(image_folder)
 image_folder = image_folder if is_test else os.path.join(data_dir, 'train_images')
 train_folder = os.path.join(data_dir, 'train_images')
 df = df_test if is_test else df_train.loc[:1]
tile_size = 256
image_size = 256
n_tiles = 36
batch_size = 12
num_workers = 12
device = torch.device('cuda')
 print(image_folder )
def matthews_correlation(y_true, y_pred):

 y_pred = tf.convert_to_tensor(y_pred, np.float32)
 y_true = tf.convert_to_tensor(y_true, np.float32)
 y_pred_pos = K.round(K.clip(y_pred, 0, 1))
y_pred_neg = 1 - y_pred_pos
y_pos = K.round(K.clip(y_true, 0, 1))
y_neg = 1 - y_pos
tp = K.sum(y_pos * y_pred_pos)
 tn = K.sum(y_neg * y_pred_neg)
 fp = K.sum(y_neg * y_pred_pos)
 fn = K.sum(y_pos * y_pred_neg)
 numerator =(tp * tn - fp * fn)
 denominator = K.sqrt(( tp + fp)*(tp + fn)*(tn + fp)*(tn + fn))
return numerator /(denominator + K.epsilon() )<choose_model_class>
class enetv2(nn.Module):
def __init__(self, backbone, out_dim):
super(enetv2, self ).__init__()
self.enet = enet.EfficientNet.from_name(backbone)
 self.myfc = nn.Linear(self.enet._fc.in_features, out_dim)
 self.enet._fc = nn.Identity()
def extract(self, x):
return self.enet(x)
 def forward(self, x):
x = self.extract(x)
 x = self.myfc(x)
 return x
def load_models(model_files):
models = []
for model_f in model_files:
model_f = os.path.join(model_dir, model_f)
 backbone = 'efficientnet-b0'
model = enetv2(backbone, out_dim=5)
 model.load_state_dict(torch.load(model_f, map_location=lambda storage, loc: storage), strict=False)
 model.eval()
model.to(device)
 models.append(model)
 print(f'{model_f} loaded!')
 return models
model_files = [
'b0-stain-bb9_best_fold0.pth','b0-stain-964_best_fold1.pth','b0-stain-917_best_fold2.pth',
'b0-stain-c11_best_fold3.pth'
]
models = load_models(model_files )
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim<normalization>
def sampleMeanStd(img):
img = img.astype("float32")
 b_ch=np.mean(img[:,:,0])
 g_ch=np.mean(img[:,:,1])
 r_ch=np.mean(img[:,:,2])
 img -= np.array(( b_ch,g_ch,r_ch))
b_ch=np.std(img[:,:,0])
 g_ch=np.std(img[:,:,1])
 r_ch=np.std(img[:,:,2])
 img /= np.array(( b_ch,g_ch,r_ch))
return img
def get_tiles(img, mode=0,forReference=False):
result = []
h, w, c = img.shape
pad_h =(tile_size - h % tile_size)% tile_size +(( tile_size * mode)// 2)
 pad_w =(tile_size - w % tile_size)% tile_size +(( tile_size * mode)// 2)
 img2 = np.pad(img,[[pad_h // 2, pad_h - pad_h // 2], [pad_w // 2,pad_w - pad_w//2], [0,0]], constant_values=255)
 img3 = img2.reshape(
 img2.shape[0] // tile_size,
tile_size,
img2.shape[1] // tile_size,
tile_size,
3 
)
 img3 = img3.transpose(0,2,1,3,4 ).reshape(-1, tile_size, tile_size,3)
 n_tiles_with_info =(img3.reshape(img3.shape[0],-1 ).sum(1)< tile_size ** 2 * 3 * 245 ).sum()
if len(img)< n_tiles:
img3 = np.pad(img3,[[0,N-len(img3)],[0,0],[0,0],[0,0]], constant_values=255)
 if forReference:
idxs = np.argsort(img3.reshape(img3.shape[0],-1 ).sum(-1)) [:n_tiles]
else:
idxs = np.argsort(img3.reshape(img3.shape[0],-1 ).sum(-1)) [:n_tiles_with_info]
img3 = img3[idxs]
for i in range(len(img3)) :
result.append({'img':img3[i], 'idx':i})
 return result, n_tiles_with_info
def getItemCustom(img_id,n_tiles=36):
tiff_file = os.path.join(train_folder, f'{img_id}.tiff')
 image = skimage.io.MultiImage(tiff_file)[1]
tiles, OK = get_tiles(image,forReference=True)
 idxes = list(range(n_tiles))
n_row_tiles = int(np.sqrt(n_tiles))
images = np.zeros(( image_size * n_row_tiles, image_size * n_row_tiles, 3))
for h in range(n_row_tiles):
for w in range(n_row_tiles):
i = h * n_row_tiles + w
if len(tiles)> idxes[i]:
this_img = tiles[idxes[i]]['img']
else:
this_img = np.ones(( self.image_size, self.image_size, 3)).astype(np.uint8)* 255
h1 = h * image_size
w1 = w * image_size
images[h1:h1+image_size, w1:w1+image_size] = this_img
return images
def squash(x, axis=-1):

 s_squared_norm = K.sum(K.square(x), axis, keepdims=True)+ K.epsilon()
scale = K.sqrt(s_squared_norm)/(0.5 + s_squared_norm)
 return scale * x
class Capsule(Layer):

 def __init__(self,
num_capsule,
dim_capsule,
routings=3,
share_weights=True,
initializer='glorot_uniform',
activation=squash,
regularizer=None,
constraint=None,
**kwargs):
super(Capsule, self ).__init__(**kwargs)
 self.num_capsule = num_capsule
self.dim_capsule = dim_capsule
self.routings = routings
self.share_weights = share_weights
self.activation = activations.get(activation)
 self.regularizer = regularizers.get(regularizer)
 self.initializer = initializers.get(initializer)
 self.constraint = constraints.get(constraint)
 def build(self, input_shape):
input_dim_capsule = input_shape[-1]
if self.share_weights:
self.W = self.add_weight(name='capsule_kernel',
shape=(1,
input_dim_capsule,
self.num_capsule *
self.dim_capsule),
initializer=self.initializer,
regularizer=self.regularizer,
constraint=self.constraint,
trainable=True)
 else:
input_num_capsule = input_shape[-2]
self.W = self.add_weight(name='capsule_kernel',
shape=(input_num_capsule,
input_dim_capsule,
self.num_capsule *
self.dim_capsule),
initializer=self.initializer,
regularizer=self.regularizer,
constraint=self.constraint,
trainable=True)
 self.build = True
def call(self, inputs):
if self.share_weights:
u_hat_vectors = K.conv1d(inputs, self.W)
 else:
u_hat_vectors = K.local_conv1d(inputs, self.W, [1], [1])
 batch_size = K.shape(inputs)[0]
input_num_capsule = K.shape(inputs)[1]
u_hat_vectors = K.reshape(u_hat_vectors,(batch_size,
input_num_capsule,
self.num_capsule,
self.dim_capsule))
u_hat_vectors = K.permute_dimensions(u_hat_vectors,(0, 2, 1, 3))
routing_weights = K.zeros_like(u_hat_vectors[:, :, :, 0])
 for i in range(self.routings):
capsule_weights = K.softmax(routing_weights, 1)
 outputs = K.batch_dot(capsule_weights, u_hat_vectors, [2, 2])
 if K.ndim(outputs)== 4:
outputs = K.sum(outputs, axis=1)
 if i < self.routings - 1:
outputs = K.l2_normalize(outputs, -1)
 routing_weights = K.batch_dot(outputs, u_hat_vectors, [2, 3])
 if K.ndim(routing_weights)== 4:
routing_weights = K.sum(routing_weights, axis=1)
 return self.activation(outputs)
 def compute_output_shape(self, input_shape):
return(None, self.num_capsule, self.dim_capsule)
 def get_config(self):
config = {'num_capsule': self.num_capsule,
'dim_capsule': self.dim_capsule,
'routings': self.routings,
'share_weights': self.share_weights,
'activation': activations.serialize(self.activation),
'regularizer': regularizers.serialize(self.regularizer),
'initializer': initializers.serialize(self.initializer),
'constraint': constraints.serialize(self.constraint)}
base_config = super(Capsule, self ).get_config()
return dict(list(base_config.items())+ list(config.items()))<load_from_csv>
reference_stain = getItemCustom('001c62abd11fa4b57bf7a6c603a11bb9')
 reference_stain = np.uint8(reference_stain)
 reference_stain = cv2.resize(reference_stain,(1024,1024))
reference_stain = staintools.LuminosityStandardizer.standardize(reference_stain)
 normalizer = staintools.StainNormalizer(method='vahadane')
 normalizer.fit(reference_stain)
 reference_stain = getItemCustom('008069b542b0439ed69b194674051964')
 reference_stain = np.uint8(reference_stain)
 reference_stain = cv2.resize(reference_stain,(1024,1024))
reference_stain = staintools.LuminosityStandardizer.standardize(reference_stain)
 normalizer2 = staintools.StainNormalizer(method='vahadane')
 normalizer2.fit(reference_stain)
 reference_stain = getItemCustom('0005f7aaab2800f6170c399693a96917')
 reference_stain = np.uint8(reference_stain)
 reference_stain = cv2.resize(reference_stain,(1024,1024))
reference_stain = staintools.LuminosityStandardizer.standardize(reference_stain)
 normalizer3 = staintools.StainNormalizer(method='vahadane')
 normalizer3.fit(reference_stain)
 reference_stain = getItemCustom('999a911f00a8647b3603859bf62c8c11')
 reference_stain = np.uint8(reference_stain)
 reference_stain = cv2.resize(reference_stain,(1024,1024))
reference_stain = staintools.LuminosityStandardizer.standardize(reference_stain)
 normalizer4 = staintools.StainNormalizer(method='vahadane')
 normalizer4.fit(reference_stain )
df_train = pd.read_csv('.. /input/metadata_train.csv')
 df_train = df_train.set_index(['id_measurement', 'phase'])
 df_train.head()<define_variables>
class PANDADataset(Dataset):
def __init__(self,
df,
image_size,
n_tiles=n_tiles,
tile_mode=0,
rand=False,
sub_imgs=False,
normalizer = None 
):
self.df = df.reset_index(drop=True)
 self.image_size = image_size
self.n_tiles = n_tiles
self.tile_mode = tile_mode
self.rand = rand
self.sub_imgs = sub_imgs
self.normalizer= normalizer
def __len__(self):
return self.df.shape[0]
def __getitem__(self, index):
row = self.df.iloc[index]
img_id = row.image_id
tiff_file = os.path.join(image_folder, f'{img_id}.tiff')
 image = skimage.io.MultiImage(tiff_file)[1]
tiles, tiles_count = get_tiles(image, self.tile_mode)
 idxes = list(range(tiles_count))
if tiles_count < 36:
idxes = list(range(self.n_tiles))
if tiles_count > 45:
idxes = list(range(self.n_tiles*2))
n_row_tiles = int(np.sqrt(self.n_tiles))
images = np.zeros(( image_size * n_row_tiles, image_size * n_row_tiles, 3))
for h in range(n_row_tiles):
for w in range(n_row_tiles):
i = h * n_row_tiles + w
if len(tiles)> idxes[i]:
this_img = tiles[idxes[i]]['img']
else:
this_img = np.ones(( self.image_size, self.image_size, 3)).astype(np.uint8)* 255
h1 = h * image_size
w1 = w * image_size
images[h1:h1+image_size, w1:w1+image_size] = this_img
if tiles_count > 45:
images2 = np.zeros(( image_size * n_row_tiles, image_size * n_row_tiles, 3))
for h in range(n_row_tiles):
for w in range(n_row_tiles):
i = h * n_row_tiles + w
if len(tiles)> idxes[i+36]:
this_img = tiles[idxes[i+36]]['img']
else:
this_img = np.ones(( self.image_size, self.image_size, 3)).astype(np.uint8)* 255
h1 = h * image_size
w1 = w * image_size
images2[h1:h1+image_size, w1:w1+image_size] = this_img
else:
images2 = np.copy(images)
 image_to_transform = np.copy(images)
 image_to_transform = np.uint8(image_to_transform)
 image_to_transform = cv2.resize(image_to_transform,(1024,1024))
try:
image_to_transform = staintools.LuminosityStandardizer.standardize(image_to_transform)
 normalized_image = self.normalizer.transform(image_to_transform)
 normalized_image = cv2.resize(normalized_image,(1536,1536))
images = normalized_image
except Exception as e:
print(e)
 images = 255 - images
images = images.astype(np.float32)
 images /= 255
images = images.transpose(2, 0, 1)
 if tiles_count > 45:
image_to_transform = np.copy(images2)
 image_to_transform = np.uint8(image_to_transform)
 image_to_transform = cv2.resize(image_to_transform,(1024,1024))
try:
image_to_transform = staintools.LuminosityStandardizer.standardize(image_to_transform)
 normalized_image = self.normalizer.transform(image_to_transform)
 normalized_image = cv2.resize(normalized_image,(1536,1536))
images2 = normalized_image
except Exception as e:
print(e)
 images2 = 255 - images2
images2 = images2.astype(np.float32)
 images2 /= 255
images2 = images2.transpose(2, 0, 1)
 else:
images2 = 255 - images2
images2 = images2.astype(np.float32)
 images2 /= 255
images2 = images2.transpose(2, 0, 1)
 return torch.tensor([images,images2])
max_num = 127
min_num = -128<categorify>
dataset = PANDADataset(df, image_size, n_tiles, tile_mode=0,normalizer=normalizer)
 loader = DataLoader(dataset, batch_size=batch_size, num_workers=num_workers, shuffle=False)
 dataset2 = PANDADataset(df, image_size, n_tiles, tile_mode=0,normalizer=normalizer2)
 loader2 = DataLoader(dataset2, batch_size=batch_size, num_workers=num_workers, shuffle=False)
 dataset3 = PANDADataset(df, image_size, n_tiles, tile_mode=0,normalizer=normalizer3)
 loader3 = DataLoader(dataset3, batch_size=batch_size, num_workers=num_workers, shuffle=False)
 dataset4 = PANDADataset(df, image_size, n_tiles, tile_mode=0,normalizer=normalizer4)
 loader4 = DataLoader(dataset4, batch_size=batch_size, num_workers=num_workers, shuffle=False )
def min_max_transformation(ts, min_data, max_data, range_needed=(-1,1)) :
if min_data < 0:
ts_std =(ts + abs(min_data)) /(max_data + abs(min_data))
else:
ts_std =(ts - min_data)/(max_data - min_data)
 if range_needed[0] < 0:
return ts_std *(range_needed[1] + abs(range_needed[0])) + range_needed[0]
else:
return ts_std *(range_needed[1] - range_needed[0])+ range_needed[0]<categorify>
from time import time
<prepare_x_and_y><EOS> 
start_time = time()
LOGITS = []
LOGITS2 = []
LOGITS3 = []
LOGITS4 = []
with torch.no_grad() :
for data in tqdm(loader):
all_logits = []
for i in range(data.shape[0]):
img1,img2 = data[i][0],data[i][1]
logits = models[0](data[i].to(device))
logits = torch.mean(logits,dim=0)
 all_logits.append(logits)
 logits = torch.stack(all_logits)
 LOGITS.append(logits)
 with torch.no_grad() :
for data in tqdm(loader2):
all_logits = []
for i in range(data.shape[0]):
img1,img2 = data[i][0],data[i][1]
logits = models[1](data[i].to(device))
logits = torch.mean(logits,dim=0)
 all_logits.append(logits)
 logits = torch.stack(all_logits)
 LOGITS2.append(logits)
 with torch.no_grad() :
for data in tqdm(loader3):
all_logits = []
for i in range(data.shape[0]):
img1,img2 = data[i][0],data[i][1]
logits = models[2](data[i].to(device))
logits = torch.mean(logits,dim=0)
 all_logits.append(logits)
 logits = torch.stack(all_logits)
 LOGITS3.append(logits)
 with torch.no_grad() :
for data in tqdm(loader4):
all_logits = []
for i in range(data.shape[0]):
img1,img2 = data[i][0],data[i][1]
logits = models[3](data[i].to(device))
logits = torch.mean(logits,dim=0)
 all_logits.append(logits)
 logits = torch.stack(all_logits)
 LOGITS4.append(logits)
 LOGITS_all =(torch.cat(LOGITS ).sigmoid().cpu() + torch.cat(LOGITS2 ).sigmoid().cpu() + torch.cat(LOGITS3 ).sigmoid().cpu() + torch.cat(LOGITS4 ).sigmoid().cpu())/ 4
PREDS = LOGITS_all.sum(1 ).round().numpy()
df['isup_grade'] = PREDS.astype(int)
 df[['image_id', 'isup_grade']].to_csv('submission.csv', index=False)
 print('Total time taken',time() -start_time)
 print(df.head())
 print()
print(df.isup_grade.value_counts() )