Kaggle competition: Prostate cANcer graDe Assessment (PANDA) Challenge
<SOS> metric: QuadraticWeightedKappa Kaggle data source: prostate-cancer-grade-assessment-(panda)-challenge<data_type_conversions>
import sys
import os
import glob
from tqdm.notebook import tqdm
import shutil
from pathlib import Path
from openslide import OpenSlide
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from skimage.measure import block_reduce
import cv2.cv2 as cv2
from sys import getsizeof
import collections
import pickle
meta_train_df['target'].mean() ,(yp_train > best_proba ).astype(np.float ).mean() , submission['target'].mean()<save_model>
physical_devices = tf.config.list_physical_devices('GPU')
 tf.config.experimental.set_memory_growth(physical_devices[0], True )
for j in range(len(models)) :
models[j].save_model('model_{}.txt'.format(j))<import_modules>
data_dir = '.. /input/prostate-cancer-grade-assessment'
df_train = pd.read_csv(os.path.join(data_dir, 'train.csv'))
df_test = pd.read_csv(os.path.join(data_dir, 'test.csv'))
df_sub = pd.read_csv(os.path.join(data_dir, 'sample_submission.csv'))
model_dir = '.. /input/panda-public-models'
image_folder = os.path.join(data_dir, 'test_images')
 is_test = os.path.exists(image_folder)
 image_folder = image_folder if is_test else os.path.join(data_dir, 'train_images')
 df = df_test if is_test else df_train.loc[:10]
df.head()
seed = 1015
np.random.seed(seed)
 tf.set_random_seed(seed)
<statistical_test>
def roll(a,
window,
dx=1,
dy=1):
shape =(
(( a.shape[0] - window[0])// dy + 1), 
(( a.shape[1] - window[1])// dx + 1),
*window, 
)
 strides =(
 a.strides[0] * dy,
a.strides[1] * dx,
a.strides[0],
a.strides[1], 
)
 return np.lib.stride_tricks.as_strided(a, shape=shape, strides=strides)
 def reduce(image, block, stride):
return roll(image,(block, block), stride, stride ).mean(( -1,-2))
def tile_filter(tile, provider):
if provider[:3] == 'rad':
if(( 1.0 - tile / 255 ).mean(2)< 0.1 ).astype(np.float ).mean() > 0.4:
return False
if(( tile / 255 ).mean(2)< 0.4 ).astype(np.float ).mean() > 0.2:
return False
else:
if(( 1.0 - tile / 255 ).mean(2)< 0.2 ).astype(np.float ).mean() > 0.3:
return False
return True
def denoise(signal, wavet = "db1", level=1):
coeff = pywt.wavedec(signal, wavet, mode="per")
 sigma = mad(coeff[-level])
 uthresh = sigma * np.sqrt(2*np.log(len(signal)))
 coeff[1:] =(pywt.threshold(i, value=uthresh, mode="hard")for i in coeff[1:])
 DY = pywt.waverec(coeff, wavet, mode="per")
 return DY
def sino(signal):
B, A = butter(1, 0.02, output='ba')
 sino_signal = filtfilt(B,A, signal)
 return signal-sino_signal
def wavelt(signal):
cA,cD = pywt.dwt(signal, "db1")
 return cA, cD<define_variables>
model = load_model('.. /input/tilenet/checkout_panda_v3.hdf5' )
N_SPLITS = 5
sample_size = 800000<compute_test_metric>
output = [model.output, model.layers[-2].output]
K = output[1].shape[1:]
mmodel = Model(inputs=model.input, outputs = output)
 print(f'
def matthews_correlation(y_true, y_pred):

 y_pred_pos = K.round(K.clip(y_pred, 0, 1))
y_pred_neg = 1 - y_pred_pos
y_pos = K.round(K.clip(y_true, 0, 1))
y_neg = 1 - y_pos
tp = K.sum(y_pos * y_pred_pos)
 tn = K.sum(y_neg * y_pred_neg)
 fp = K.sum(y_neg * y_pred_pos)
 fn = K.sum(y_pos * y_pred_neg)
 numerator =(tp * tn - fp * fn)
 denominator = K.sqrt(( tp + fp)*(tp + fn)*(tn + fp)*(tn + fn))
return numerator /(denominator + K.epsilon() )<choose_model_class>
class Conv2dBlock(nn.Module):
def __init__(self, in_channels: int, out_channels: int, layers: int = 2,
activation: Type[nn.Module] = nn.ReLU, kernel_size: int = 3,
batchnorm: bool = True):
super(Conv2dBlock, self ).__init__()
self.blocks = []
for i in range(layers):
block_layers = [
nn.Conv2d(( in_channels if i == 0 else out_channels),
out_channels, kernel_size, padding=kernel_size // 2),
activation()
]
if batchnorm:
block_layers.insert(1, nn.BatchNorm2d(out_channels))
block = nn.Sequential(*block_layers)
 self.blocks.append(block)
 self.main = nn.Sequential(*self.blocks)
 def forward(self, inputs: Tensor)-> Tensor:

 return self.main(inputs)
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim<load_from_csv>
class PandaModel(nn.Module):
def __init__(self, n_channels: int, n_classes: Union[int, Tuple[int,...]]):
super().__init__()
self.encoder = nn.Sequential(
 Conv2dBlock(n_channels, 1024),
Conv2dBlock(1024, 512),
Conv2dBlock(512, 256),
Conv2dBlock(256, 128), 
)
 def make_head(n: int):
return nn.Sequential(
 nn.Linear(128, 32),
nn.ReLU() ,
nn.Linear(32, 16),
nn.ReLU() ,
nn.Linear(16, n)
)
 if isinstance(n_classes, Iterable):
self.heads = nn.ModuleList([make_head(n)for n in n_classes])
 else:
self.heads = nn.ModuleList([make_head(n_classes)])
 def forward(self, x):
d = self.encoder(x ).detach()
d = d.max(3 ).values.max(2 ).values.detach()
res = [h(d ).detach() for h in self.heads]
return res if len(res)> 1 else res[0]
df_train = pd.read_csv('.. /input/metadata_train.csv')
 df_train = df_train.set_index(['id_measurement', 'phase'])
 df_train.head()<define_variables>
wsi_model = PandaModel(2048, 6)
 wsi_model.load_state_dict(torch.load('.. /input/wsinet/best_qp2_723.pth', map_location='cpu')['model'] )
max_num = 127
min_num = -128<categorify>
def find_stain_index(reference, w):
dot_products = np.dot(reference, w)
 return np.argmax(np.abs(dot_products))
def min_max_transf(ts, min_data, max_data, range_needed=(-1,1)) :
if min_data < 0:
ts_std =(ts + abs(min_data)) /(max_data + abs(min_data))
else:
ts_std =(ts - min_data)/(max_data - min_data)
 if range_needed[0] < 0:
return ts_std *(range_needed[1] + abs(range_needed[0])) + range_needed[0]
else:
return ts_std *(range_needed[1] - range_needed[0])+ range_needed[0]<categorify>
def _reorder_stains(W, stains=None):
stains = ['hematoxylin', 'eosin'] if stains is None else stains
assert len(stains)== 2, "Only two-stain matrices are supported for now."
def _get_channel_order(W):
first = find_stain_index(stain_color_map[stains[0]], W)
 second = 1 - first
third = 2
return first, second, third
def _ordered_stack(mat, order):
return np.stack([mat[..., j] for j in order], -1)
 return _ordered_stack(W, _get_channel_order(W))
def transform_ts(ts, n_dim=160, min_max=(-1,1)) :
ts = denoise(ts)
 ts_std = min_max_transf(ts, min_data=min_num, max_data=max_num)
 bucket_size = int(sample_size / n_dim)
 new_ts = []
for i in range(0, sample_size, bucket_size):
ts_range = ts_std[i:i + bucket_size]
mean = ts_range.mean()
std = ts_range.std()
std_top = mean + std
std_bot = mean - std
percentil_calc = np.percentile(ts_range, [0, 1, 25, 50, 75, 99, 100])
 max_range = percentil_calc[-1] - percentil_calc[0]
relative_percentile = percentil_calc - mean
new_ts.append(np.concatenate([np.asarray([mean, std, std_top, std_bot, max_range]),percentil_calc, relative_percentile]))
return np.asarray(new_ts )<prepare_x_and_y>
def convert_image_to_matrix(im):
if im.ndim == 2:
return im
return im.reshape(( -1, im.shape[-1])).T
def convert_matrix_to_image(m, shape):
if len(shape)== 2:
return m
return m.T.reshape(shape[:-1] +(m.shape[0],))
def get_principal_components(m):
return numpy.linalg.svd(m.astype(float), full_matrices=False)[0].astype(m.dtype)
 def magnitude(m):
return numpy.sqrt(( m ** 2 ).sum(0))
def normalize(m):
return m / magnitude(m )
def prep_data(start, end):
praq_train = pq.read_pandas('.. /input/train.parquet', columns=[str(i)for i in range(start, end)] ).to_pandas()
X = []
y = []
for id_measurement in tqdm(df_train.index.levels[0].unique() [int(start/3):int(end/3)]):
X_signal = []
for phase in [0,1,2]:
signal_id, target = df_train.loc[id_measurement].loc[phase]
if phase == 0:
y.append(target)
 X_signal.append(transform_ts(praq_train[str(signal_id)]))
X_signal = np.concatenate(X_signal, axis=1)
 X.append(X_signal)
 X = np.asarray(X)
 y = np.asarray(y)
 return X, y<concatenate>
def rgb_to_sda(im_rgb, I_0, allow_negatives=False):
is_matrix = im_rgb.ndim == 2
if is_matrix:
im_rgb = im_rgb.T
if I_0 is None:
im_rgb = im_rgb.astype(float)+ 1
I_0 = 256
if not allow_negatives:
im_rgb = np.minimum(im_rgb, I_0)
 im_sda = -np.log(im_rgb/(1.*I_0)) * 255/np.log(I_0)
 return im_sda.T if is_matrix else im_sda
def sda_to_rgb(im_sda, I_0):
is_matrix = im_sda.ndim == 2
if is_matrix:
im_sda = im_sda.T
od = I_0 is None
if od:
I_0 = 256
im_rgb = I_0 **(1 - im_sda / 255.)
return(im_rgb.T if is_matrix else im_rgb)- od
def exclude_nonfinite(m):
return m[:, np.isfinite(m ).all(axis=0)]
X = []
y = []
def load_all() :
total_size = len(df_train)
 for ini, end in [(0, int(total_size/2)) ,(int(total_size/2), total_size)]:
X_temp, y_temp = prep_data(ini, end)
 X.append(X_temp)
 y.append(y_temp)
 load_all()
X = np.concatenate(X)
 y = np.concatenate(y )<choose_model_class>
def color_convolution(im_stains, w, I_0=None):
m = convert_image_to_matrix(im_stains)
 sda_fwd = rgb_to_sda(m, 255 if I_0 is not None else None,allow_negatives=True)
 sda_conv = np.dot(w, sda_fwd)
 sda_inv = sda_to_rgb(sda_conv, I_0)
 im_rgb =(convert_matrix_to_image(sda_inv, im_stains.shape ).clip(0, 255 ).astype(np.uint8))
return im_rgb
def model_lstm(input_shape):
inp = Input(shape=(input_shape[1], input_shape[2],))
x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(inp)
 x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)
 x = Attention(input_shape[1] )(x)
 x = Dense(64, activation="relu" )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=[matthews_correlation])
 return model
def model_(input_shape):
model = Sequential()
model.add(Conv2D(24, kernel_size=3,input_shape=(input_shape[1],input_shape[2],1)))
 model.add(ReLU())
 model.add(AveragePooling2D())
 model.add(Flatten())
 model.add(Dropout(0.7))
model.add(Dense(128))
model.add(BatchNormalization())
 model.add(ReLU())
 model.add(Dropout(0.7))
model.add(Dense(148))
model.add(ReLU())
 model.add(Dropout(0.7))
model.add(Dense(1, activation='sigmoid'))
model.compile(loss='binary_crossentropy', optimizer="adam", metrics=[matthews_correlation])
 return model
def model_lstm2(input_shape):
inp = Input(shape=(input_shape[1], input_shape[2],))
x = Bidirectional(CuDNNGRU(128, return_sequences=True))(inp)
 x = Bidirectional(CuDNNGRU(64))(x)
 x = Attention(input_shape[1] )(x)
 x = Dense(64, activation="relu" )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=[matthews_correlation])
 return model<split>
def complement_stain_matrix(w):
stain0 = w[:, 0]
stain1 = w[:, 1]
stain2 = np.cross(stain0, stain1)
 return np.array([stain0, stain1, stain2 / np.linalg.norm(stain2)] ).T
splits = list(StratifiedKFold(n_splits=N_SPLITS, shuffle=True, random_state=2019 ).split(X, y))
preds_val = []
y_val = []
histories = []
for idx,(train_idx, val_idx)in enumerate(splits):
K.clear_session()
print("Beginning fold {}".format(idx+1))
train_X, train_y, val_X, val_y = X[train_idx], y[train_idx], X[val_idx], y[val_idx]
model = model_lstm(train_X.shape)
 ckpt = ModelCheckpoint('weights_{}.h5'.format(idx), save_best_only=True, save_weights_only=True, verbose=1, monitor='val_matthews_correlation', mode='max')
 h = model.fit(train_X, train_y, batch_size=128, epochs=50, validation_data=[val_X, val_y], callbacks=[ckpt])
 model.load_weights('weights_{}.h5'.format(idx))
preds_val.append(model.predict(val_X, batch_size=512))
y_val.append(val_y)
 histories.append(h)
 preds_val = np.concatenate(preds_val)[...,0]
y_val = np.concatenate(y_val)
 preds_val.shape, y_val.shape<find_best_params>
def rgb_separate_stains_macenko_pca(im_rgb, I_0, *args, **kwargs):
im_sda = rgb_to_sda(im_rgb, I_0)
 return separate_stains_macenko_pca(im_sda, *args, **kwargs )
def threshold_search(y_true, y_proba):
best_threshold = 0
best_score = 0
for threshold in tqdm([i * 0.01 for i in range(100)]):
score = K.eval(matthews_correlation(y_true.astype(np.float64),(y_proba > threshold ).astype(np.float64)))
 if score > best_score:
best_threshold = threshold
best_score = score
search_result = {'threshold': best_threshold, 'matthews_correlation': best_score}
return search_result<compute_test_metric>
def separate_stains_macenko_pca(
 im_sda, minimum_magnitude=16, min_angle_percentile=0.01,
max_angle_percentile=0.99, mask_out=None):
m = convert_image_to_matrix(im_sda)
 if mask_out is not None:
keep_mask = numpy.equal(mask_out[..., None], False)
 keep_mask = numpy.tile(keep_mask,(1, 1, 3))
keep_mask = convert_image_to_matrix(keep_mask)
 m = m[:, keep_mask.all(axis=0)]
m = exclude_nonfinite(m)
 pcs = get_principal_components(m)
 proj = pcs.T[:-1].dot(m)
 filt = proj[:, magnitude(proj)> minimum_magnitude]
angles = _get_angles(filt)
 def get_percentile_vector(p):
return pcs[:, :-1].dot(filt[:, argpercentile(angles, p)])
 min_v = get_percentile_vector(min_angle_percentile)
 max_v = get_percentile_vector(max_angle_percentile)
 w = complement_stain_matrix(normalize(numpy.array([min_v, max_v] ).T))
return w
def _get_angles(m):
m = normalize(m)
 return(1 - m[1])* numpy.sign(m[0])
 def argpercentile(arr, p):

 i = int(p * arr.size + 0.5)
 return numpy.argpartition(arr, i)[i]
best_threshold = threshold_search(y_val, preds_val)['threshold']<load_from_csv>
stain_unmixing_routine_params = {
'stains': ['hematoxylin', 'eosin'],
'stain_unmixing_method': 'macenko_pca',
}
def dbn(im_src, W_source=None, W_target=None, im_target=None,stains=None, mask_out=None, stain_unmixing_routine_params=None):
stains = ['hematoxylin', 'eosin'] if stains is None else stains
stain_unmixing_routine_params =(
 {} if stain_unmixing_routine_params is None else
stain_unmixing_routine_params)
 for k in ['W_source', 'mask_out']:
assert k not in stain_unmixing_routine_params.keys() , \
"%s must be provided as a separate parameter." % k
stain_unmixing_routine_params['stains'] = stains
_, StainsFloat, _ = color_deconvolution_routine(
 im_src, W_source=W_source, mask_out=mask_out,
**stain_unmixing_routine_params)
 if all(j is None for j in [W_target, im_target]):
W_target = np.array(
 [stain_color_map[stains[0]], stain_color_map[stains[1]]] ).T
W_target = complement_stain_matrix(W_target)
 elif im_target is not None:
W_target = stain_unmixing_routine(
 im_target, **stain_unmixing_routine_params)
 im_src_normalized = color_convolution(StainsFloat, W_target)
 if mask_out is not None:
keep_mask = np.not_equal(mask_out, True)
 for i in range(3):
original = im_src[:, :, i].copy()
new = im_src_normalized[:, :, i].copy()
original[keep_mask] = 0
new[mask_out] = 0
im_src_normalized[:, :, i] = new + original
return im_src_normalized
%%time
meta_test = pd.read_csv('.. /input/metadata_test.csv' )<drop_column>
LEVEL = 1
TILE_SIZE = 256
res_isup = np.zeros(len(df), dtype=np.int )
<define_variables><EOS> 
df['isup_grade'] = np.array(res_isup, dtype=np.int)
 df[['image_id', 'isup_grade']].to_csv('submission.csv', index=False)
 print(df.head())
 print()
print(df.isup_grade.value_counts() )