Kaggle competition: Prostate cANcer graDe Assessment (PANDA) Challenge
<SOS> metric: QuadraticWeightedKappa Kaggle data source: prostate-cancer-grade-assessment-(panda)-challenge<categorify>
DEBUG = False
@numba.jit(nopython=True)
 def plateau_detection(grad, threshold, plateau_length=5):

 count = 0
loc = 0
for i in range(grad.shape[0]):
if grad[i] > threshold:
count += 1
if count == plateau_length:
loc = i - plateau_length
break
return loc<categorify>
sys.path = [
'.. /input/efficientnet-pytorch/EfficientNet-PyTorch/EfficientNet-PyTorch-master',
] + sys.path
def get_peaks(
 x,
window=25,
visualise=False,
visualise_color=None, 
):

 x_hp = flatiron(x, 100, 1)
 x_dn = np.abs(x_hp)
 peaks = local_maxima_1d_window(x_dn, window)
 heights = x_dn[peaks]
ii = np.argsort(heights)[::-1]
peaks = peaks[ii]
heights = heights[ii]
ky = heights
kx = np.arange(1, heights.shape[0]+1)
 conv_length = 9
grad = np.diff(ky, 1)/np.diff(kx, 1)
 grad = np.convolve(grad, np.ones(conv_length)/conv_length)
 grad = grad[conv_length-1:-conv_length+1]
knee_x = plateau_detection(grad, -0.01, plateau_length=1000)
 knee_x -= conv_length//2
if visualise:
plt.plot(grad, color=visualise_color)
 plt.axvline(knee_x, ls="--", color=visualise_color)
 peaks_x = peaks[:knee_x]
peaks_y = heights[:knee_x]
ii = np.argsort(peaks_x)
 peaks_x = peaks_x[ii]
peaks_y = peaks_y[ii]
return peaks_x, peaks_y, x_hp<train_model>
import skimage.io
import numpy as np
import pandas as pd
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.utils.data import DataLoader, Dataset
from efficientnet_pytorch import model as enet
import matplotlib.pyplot as plt
from tqdm import tqdm_notebook as tqdm
@numba.jit(nopython=True)
 def clip(v, l, u):

 if v < l:
v = l
elif v > u:
v = u
return v<categorify>
data_dir = '.. /input/prostate-cancer-grade-assessment'
df_train = pd.read_csv(os.path.join(data_dir, 'train.csv'))
df_test = pd.read_csv(os.path.join(data_dir, 'test.csv'))
df_sub = pd.read_csv(os.path.join(data_dir, 'sample_submission.csv'))
model_dir = '.. /input/panda-public-models'
image_folder = os.path.join(data_dir, 'test_images')
 is_test = os.path.exists(image_folder)
 image_folder = image_folder if is_test else os.path.join(data_dir, 'train_images')
 df = df_test if is_test else df_train.loc[:10]
tile_size = 256
image_size = 256
n_tiles = 36
batch_size = 8
num_workers = 4
device = torch.device('cuda')
 print(image_folder)
peak_features_names = [
'ratio_next',
'ratio_prev',
'small_dist_to_min',
'sawtooth_rmse',
]
num_peak_features = len(peak_features_names)
 @numba.jit(nopython=True)
 def create_sawtooth_template(sawtooth_length, pre_length, post_length):

 l = pre_length+post_length+1
st = np.zeros(l)
 for i in range(sawtooth_length+1):
j = pre_length+i
if j < l:
st[j] = 1 -(( 2./sawtooth_length)* i)
 return st
@numba.jit(nopython=True)
 def calculate_peak_features(px, x_hp0, ws=5, wl=25):

 features = np.ones(( px.shape[0], num_peak_features), dtype=np.float64)* np.nan
for i in range(px.shape[0]):
feature_number = 0
x = px[i]
x_next = x+1
x_prev = x-1
h0 = x_hp0[x]
ws_s = clip(x-ws, 0, 800000-1)
 ws_e = clip(x+ws, 0, 800000-1)
 wl_s = clip(x-wl, 0, 800000-1)
 wl_e = clip(x+wl, 0, 800000-1)
 ws_pre = x - ws_s
ws_post = ws_e - x
wl_pre = x - wl_s
wl_post = wl_e - x
if x_next < 800000:
h0_next = x_hp0[x_next]
features[i, feature_number] = np.abs(h0_next)/np.abs(h0)
 feature_number += 1
if x_prev >= 0:
h0_prev = x_hp0[x_prev]
features[i, feature_number] = np.abs(h0_prev)/np.abs(h0)
 feature_number += 1
x_hp_ws0 = x_hp0[ws_s:ws_e+1]
x_hp_wl0 = x_hp0[wl_s:wl_e+1]
x_hp_wl0_norm =(x_hp_wl0/np.abs(h0))
x_hp_ws0_norm =(x_hp_ws0/np.abs(h0))
x_hp_abs_wl0 = np.abs(x_hp_wl0)
 wl_max_0 = np.max(x_hp_abs_wl0)
 ws_opp_peak_i = np.argmin(x_hp_ws0*np.sign(h0))
features[i, feature_number] = ws_opp_peak_i - ws
feature_number += 1
x_hp_wl0_norm_sign = x_hp_wl0_norm * np.sign(h0)
 sawtooth_length = 3
st = create_sawtooth_template(sawtooth_length, wl_pre, wl_post)
 assert np.argmax(st)== np.argmax(x_hp_wl0_norm_sign)
 assert st.shape[0] == x_hp_wl0_norm_sign.shape[0]
features[i, feature_number] = np.mean(np.power(x_hp_wl0_norm_sign - st, 2))
feature_number += 1
if i == 0:
assert feature_number == num_peak_features
return features<categorify>
class enetv2(nn.Module):
def __init__(self, backbone, out_dim):
super(enetv2, self ).__init__()
self.enet = enet.EfficientNet.from_name(backbone)
 self.myfc = nn.Linear(self.enet._fc.in_features, out_dim)
 self.enet._fc = nn.Identity()
def extract(self, x):
return self.enet(x)
 def forward(self, x):
x = self.extract(x)
 x = self.myfc(x)
 return x
def load_models(model_files):
models = []
for model_f in model_files:
model_f = os.path.join(model_dir, model_f)
 backbone = 'efficientnet-b0'
model = enetv2(backbone, out_dim=5)
 model.load_state_dict(torch.load(model_f, map_location=lambda storage, loc: storage), strict=True)
 model.eval()
model.to(device)
 models.append(model)
 print(f'{model_f} loaded!')
 return models
model_files = [
'cls_effnet_b0_Rand36r36tiles256_big_bce_lr0.3_augx2_30epo_model_fold0.pth'
]
models = load_models(model_files)
 model_dir2 = '.. /input/pandaenetb042x256x256x3'
def load_models2(model_files):
models = []
for model_f in model_files:
model_f = os.path.join(model_dir2, model_f)
 backbone = 'efficientnet-b1'
model = enetv2(backbone, out_dim=5)
 model.load_state_dict(torch.load(model_f, map_location=lambda storage, loc: storage), strict=True)
 model.eval()
model.to(device)
 models.append(model)
 print(f'{model_f} loaded!')
 return models
model_files2 = [
'enet_b1_8ep_fold0.pth'
]
models2 = load_models2(model_files2 )
def process_signal(
 data,
window=25, 
):

 px0, height0, x_hp0 = get_peaks(
 data.astype(np.float),
window=window, 
)
 f0 = calculate_peak_features(px0, x_hp0)
 return px0, height0, f0<categorify>
def get_tiles(img, mode=0):
result = []
h, w, c = img.shape
pad_h =(tile_size - h % tile_size)% tile_size +(( tile_size * mode)// 2)
 pad_w =(tile_size - w % tile_size)% tile_size +(( tile_size * mode)// 2)
 img2 = np.pad(img,[[pad_h // 2, pad_h - pad_h // 2], [pad_w // 2,pad_w - pad_w//2], [0,0]], constant_values=255)
 img3 = img2.reshape(
 img2.shape[0] // tile_size,
tile_size,
img2.shape[1] // tile_size,
tile_size,
3 
)
 img3 = img3.transpose(0,2,1,3,4 ).reshape(-1, tile_size, tile_size,3)
 n_tiles_with_info =(img3.reshape(img3.shape[0],-1 ).sum(1)< tile_size ** 2 * 3 * 255 ).sum()
if len(img)< n_tiles:
img3 = np.pad(img3,[[0,n_tiles-len(img3)],[0,0],[0,0],[0,0]], constant_values=255)
 idxs = np.argsort(img3.reshape(img3.shape[0],-1 ).sum(-1)) [:n_tiles]
img3 = img3[idxs]
for i in range(len(img3)) :
result.append({'img':img3[i], 'idx':i})
 return result, n_tiles_with_info >= n_tiles
class PANDADataset(Dataset):
def __init__(self,
df,
image_size,
n_tiles=n_tiles,
tile_mode=0,
rand=False,
sub_imgs=False,
transform=None 
):
self.df = df.reset_index(drop=True)
 self.image_size = image_size
self.n_tiles = n_tiles
self.tile_mode = tile_mode
self.rand = rand
self.sub_imgs = sub_imgs
self.transform = transform
def __len__(self):
return self.df.shape[0]
def __getitem__(self, index):
row = self.df.iloc[index]
img_id = row.image_id
tiff_file = os.path.join(image_folder, f'{img_id}.tiff')
 image = skimage.io.MultiImage(tiff_file)[1]
tiles, OK = get_tiles(image, self.tile_mode)
 if self.rand:
idxes = np.random.choice(list(range(self.n_tiles)) , self.n_tiles, replace=False)
 else:
idxes = list(range(self.n_tiles))
idxes = np.asarray(idxes)+ self.n_tiles if self.sub_imgs else idxes
n_row_tiles = int(np.sqrt(self.n_tiles))
images = np.zeros(( image_size * n_row_tiles, image_size * n_row_tiles, 3))
for h in range(n_row_tiles):
for w in range(n_row_tiles):
i = h * n_row_tiles + w
if len(tiles)> idxes[i]:
this_img = tiles[idxes[i]]['img']
else:
this_img = np.ones(( self.image_size, self.image_size, 3)).astype(np.uint8)* 255
this_img = 255 - this_img
h1 = h * image_size
w1 = w * image_size
images[h1:h1+image_size, w1:w1+image_size] = this_img
if self.transform is not None:
images = self.transform(image=images)['image']
images = images.astype(np.float32)
 images /= 255
images = images.transpose(2, 0, 1)
 return torch.tensor(images)
def process_measurement_peaks(data, signal_ids):

 res = []
sigid_res = []
assert data.shape[1] % 3 == 0
N = data.shape[1]//3
for i in range(N):
sigids = signal_ids[i*3:(i+1)*3]
x = data[:, i*3:(i+1)*3].astype(np.float)
 px0, height0, f0 = process_signal(
 x[:, 0] 
)
 px1, height1, f1 = process_signal(
 x[:, 1] 
)
 px2, height2, f2 = process_signal(
 x[:, 2] 
)
 if px0.shape[0] != 0:
res.append(np.hstack([
px0[:, np.newaxis],
height0[:, np.newaxis],
f0,
]))
sigid_res.append(np.ones(px0.shape[0], dtype=np.int)* sigids[0])
 if px1.shape[0] != 0:
res.append(np.hstack([
px1[:, np.newaxis],
height1[:, np.newaxis],
f1,
]))
sigid_res.append(np.ones(px1.shape[0], dtype=np.int)* sigids[1])
 if px2.shape[0] != 0:
res.append(np.hstack([
px2[:, np.newaxis],
height2[:, np.newaxis],
f2,
]))
sigid_res.append(np.ones(px2.shape[0], dtype=np.int)* sigids[2])
 return res, sigid_res<categorify>
transforms_train = albumentations.Compose([
albumentations.Transpose(p=0.5),
albumentations.VerticalFlip(p=0.5),
albumentations.HorizontalFlip(p=0.5),
])
 transforms_val = albumentations.Compose([])
 transforms_val1 = albumentations.Compose([
albumentations.Transpose(p=1)
 ])
 transforms_val2 = albumentations.Compose([
albumentations.VerticalFlip(p=1)
 ])
 transforms_val3= albumentations.Compose([
albumentations.HorizontalFlip(p=1),
])
 transforms_val4= albumentations.Compose([
albumentations.Transpose(p=1),
albumentations.VerticalFlip(p=1),
albumentations.HorizontalFlip(p=1),
] )
def process_measurement(data_df, meta_df, fft_data):

 peaks, sigids = process_measurement_peaks(
 data_df.values,
meta_df['signal_id'].values, 
)
 peaks = np.concatenate(peaks)
 peaks = pd.DataFrame(
 peaks,
columns=['px', 'height'] + peak_features_names 
)
 peaks['signal_id'] = np.concatenate(sigids)
 phase_50hz = np.angle(fft_data, deg=False)
 phase_50hz = pd.DataFrame(
 phase_50hz,
columns=['phase_50hz'] 
)
 phase_50hz['signal_id'] = meta_df['signal_id'].values
peaks = pd.merge(peaks, phase_50hz, on='signal_id', how='left')
 dt =(20e-3/(800000))
f1 = 50
w1 = 2*np.pi*f1
peaks['phase_aligned_x'] =(np.degrees(
(w1*peaks['px'].values*dt)+ peaks['phase_50hz'].values 
)+ 90)% 360
peaks['Q'] = pd.cut(peaks['phase_aligned_x'], [0, 90, 180, 270, 360], labels=[0, 1, 2, 3])
 return peaks<feature_engineering>
dataset = PANDADataset(df, image_size, n_tiles, 0, False, False, transforms_val)
 loader = DataLoader(dataset, batch_size=batch_size, num_workers=num_workers, shuffle=False)
 dataset2 = PANDADataset(df, image_size, n_tiles, 2, False, False, transforms_val)
 loader2 = DataLoader(dataset2, batch_size=batch_size, num_workers=num_workers, shuffle=False)
 LOGITS = []
LOGITS2 = []
LOGITS3 = []
LOGITS4 = []
with torch.no_grad() :
for data in tqdm(loader):
data = data.to(device)
 logits = models[0](data)
 LOGITS.append(logits)
 logits = models2[0](data)
 LOGITS3.append(logits)
 for data in tqdm(loader2):
data = data.to(device)
 logits = models[0](data)
 LOGITS2.append(logits)
 logits = models2[0](data)
 LOGITS4.append(logits)
 LOGITS =(torch.cat(LOGITS ).sigmoid().cpu() +torch.cat(LOGITS2 ).sigmoid().cpu() +torch.cat(LOGITS3 ).sigmoid().cpu() +torch.cat(LOGITS4 ).sigmoid().cpu())/ 4
PREDS = LOGITS.sum(1 ).numpy()
@numba.jit(nopython=True, parallel=True)
 def calculate_50hz_fourier_coefficient(data):

 n = 800000
assert data.shape[0] == n
omegas = np.exp(-2j * np.pi * np.arange(n)/ n ).reshape(n, 1)
 m_ = omegas ** np.arange(1, 2)
 m = m_.flatten()
res = np.zeros(data.shape[1], dtype=m.dtype)
 for i in numba.prange(data.shape[1]):
res[i] = m.dot(data[:, i].astype(m.dtype))
return res<normalization>
dataset = PANDADataset(df, image_size, n_tiles, 0, False, False, transforms_val1)
 loader = DataLoader(dataset, batch_size=batch_size, num_workers=num_workers, shuffle=False)
 dataset2 = PANDADataset(df, image_size, n_tiles, 2, False, False, transforms_val1)
 loader2 = DataLoader(dataset2, batch_size=batch_size, num_workers=num_workers, shuffle=False)
 LOGITS = []
LOGITS2 = []
LOGITS3 = []
LOGITS4 = []
with torch.no_grad() :
for data in tqdm(loader):
data = data.to(device)
 logits = models[0](data)
 LOGITS.append(logits)
 logits = models2[0](data)
 LOGITS3.append(logits)
 for data in tqdm(loader2):
data = data.to(device)
 logits = models[0](data)
 LOGITS2.append(logits)
 logits = models2[0](data)
 LOGITS4.append(logits)
 LOGITS =(torch.cat(LOGITS ).sigmoid().cpu() +torch.cat(LOGITS2 ).sigmoid().cpu() +torch.cat(LOGITS3 ).sigmoid().cpu() +torch.cat(LOGITS4 ).sigmoid().cpu())/ 4
PREDS1 = LOGITS.sum(1 ).numpy()
%%time
if RECALCULATE_PEAK_FEATURES:
train_df = pq.read_pandas(
 data_dir + '/train.parquet'
).to_pandas()
train_fft = calculate_50hz_fourier_coefficient(train_df.values)
 train_peaks = process_measurement(
 train_df,
meta_train_df,
train_fft 
)
 del train_df, train_fft
gc.collect()<load_pretrained>
dataset = PANDADataset(df, image_size, n_tiles, 0, False, False, transforms_val2)
 loader = DataLoader(dataset, batch_size=batch_size, num_workers=num_workers, shuffle=False)
 dataset2 = PANDADataset(df, image_size, n_tiles, 2, False, False, transforms_val2)
 loader2 = DataLoader(dataset2, batch_size=batch_size, num_workers=num_workers, shuffle=False)
 LOGITS = []
LOGITS2 = []
LOGITS3 = []
LOGITS4 = []
with torch.no_grad() :
for data in tqdm(loader):
data = data.to(device)
 logits = models[0](data)
 LOGITS.append(logits)
 logits = models2[0](data)
 LOGITS3.append(logits)
 for data in tqdm(loader2):
data = data.to(device)
 logits = models[0](data)
 LOGITS2.append(logits)
 logits = models2[0](data)
 LOGITS4.append(logits)
 LOGITS =(torch.cat(LOGITS ).sigmoid().cpu() +torch.cat(LOGITS2 ).sigmoid().cpu() +torch.cat(LOGITS3 ).sigmoid().cpu() +torch.cat(LOGITS4 ).sigmoid().cpu())/ 4
PREDS2 = LOGITS.sum(1 ).numpy()
if not RECALCULATE_PEAK_FEATURES:
train_peaks = pd.read_pickle(PREPROCESSED_DIR + '/train_peaks.pkl')
 train_peaks.to_pickle("train_peaks.pkl" )<merge>
dataset = PANDADataset(df, image_size, n_tiles, 0, False, False, transforms_val3)
 loader = DataLoader(dataset, batch_size=batch_size, num_workers=num_workers, shuffle=False)
 dataset2 = PANDADataset(df, image_size, n_tiles, 2, False, False, transforms_val3)
 loader2 = DataLoader(dataset2, batch_size=batch_size, num_workers=num_workers, shuffle=False)
 LOGITS = []
LOGITS2 = []
LOGITS3 = []
LOGITS4 = []
with torch.no_grad() :
for data in tqdm(loader):
data = data.to(device)
 logits = models[0](data)
 LOGITS.append(logits)
 logits = models2[0](data)
 LOGITS3.append(logits)
 for data in tqdm(loader2):
data = data.to(device)
 logits = models[0](data)
 LOGITS2.append(logits)
 logits = models2[0](data)
 LOGITS4.append(logits)
 LOGITS =(torch.cat(LOGITS ).sigmoid().cpu() +torch.cat(LOGITS2 ).sigmoid().cpu() +torch.cat(LOGITS3 ).sigmoid().cpu() +torch.cat(LOGITS4 ).sigmoid().cpu())/ 4
PREDS3 = LOGITS.sum(1 ).numpy()
train_peaks = pd.merge(train_peaks, meta_train_df[['signal_id', 'id_measurement', 'target']], on='signal_id', how='left' )<define_variables>
dataset = PANDADataset(df, image_size, n_tiles, 0, False, False, transforms_val4)
 loader = DataLoader(dataset, batch_size=batch_size, num_workers=num_workers, shuffle=False)
 dataset2 = PANDADataset(df, image_size, n_tiles, 2, False, False, transforms_val4)
 loader2 = DataLoader(dataset2, batch_size=batch_size, num_workers=num_workers, shuffle=False)
 LOGITS = []
LOGITS2 = []
LOGITS3 = []
LOGITS4 = []
with torch.no_grad() :
for data in tqdm(loader):
data = data.to(device)
 logits = models[0](data)
 LOGITS.append(logits)
 logits = models2[0](data)
 LOGITS3.append(logits)
 for data in tqdm(loader2):
data = data.to(device)
 logits = models[0](data)
 LOGITS2.append(logits)
 logits = models2[0](data)
 LOGITS4.append(logits)
 LOGITS =(torch.cat(LOGITS ).sigmoid().cpu() +torch.cat(LOGITS2 ).sigmoid().cpu() +torch.cat(LOGITS3 ).sigmoid().cpu() +torch.cat(LOGITS4 ).sigmoid().cpu())/ 4
PREDS4 = LOGITS.sum(1 ).numpy()
%%time
if RECALCULATE_PEAK_FEATURES:
NUM_TEST_CHUNKS = 10
test_chunk_size = int(np.ceil(( meta_test_df.shape[0]/3.) /float(NUM_TEST_CHUNKS)) *3.)
test_peaks = []
for j in range(NUM_TEST_CHUNKS):
j_start = j*test_chunk_size
j_end =(j+1)*test_chunk_size
signal_ids = meta_test_df['signal_id'].values[j_start:j_end]
test_df = pq.read_pandas(
 data_dir + '/test.parquet',
columns=[str(c)for c in signal_ids]
).to_pandas()
test_fft = calculate_50hz_fourier_coefficient(test_df.values)
 p = process_measurement(
 test_df,
meta_test_df.iloc[j_start:j_end],
test_fft 
)
 test_peaks.append(p)
 print(j)
 del test_df
gc.collect()
del test_fft
gc.collect()
test_peaks = pd.concat(test_peaks)
<load_pretrained>
!pip install.. /input/kaggle-efficientnet-repo/efficientnet-1.0.0-py3-none-any.whl
gc.enable()
sz = 256
N = 48
def tile(img):
result = []
shape = img.shape
pad0,pad1 =(sz - shape[0]%sz)%sz,(sz - shape[1]%sz)%sz
img = np.pad(img,[[pad0//2,pad0-pad0//2],[pad1//2,pad1-pad1//2],[0,0]],constant_values=255)
 img = img.reshape(img.shape[0]//sz,sz,img.shape[1]//sz,sz,3)
 img = img.transpose(0,2,1,3,4 ).reshape(-1,sz,sz,3)
 if len(img)< N:
img = np.pad(img,[[0,N-len(img)],[0,0],[0,0],[0,0]],constant_values=255)
 idxs = np.argsort(img.reshape(img.shape[0],-1 ).sum(-1)) [:N]
img = img[idxs]
return img
def tile2(img):
result = []
shape = img.shape
pad0,pad1 =(sz - shape[0]%sz)%sz +(( sz * 2)// 2),(sz - shape[1]%sz)%sz +(( sz * 2)// 2)
 img = np.pad(img,[[pad0//2,pad0-pad0//2],[pad1//2,pad1-pad1//2],[0,0]],constant_values=255)
 img = img.reshape(img.shape[0]//sz,sz,img.shape[1]//sz,sz,3)
 img = img.transpose(0,2,1,3,4 ).reshape(-1,sz,sz,3)
 if len(img)< N:
img = np.pad(img,[[0,N-len(img)],[0,0],[0,0],[0,0]],constant_values=255)
 idxs = np.argsort(img.reshape(img.shape[0],-1 ).sum(-1)) [:N]
img = img[idxs]
return img
class ConvNet(tf.keras.Model):
def __init__(self, engine, input_shape, weights):
super(ConvNet, self ).__init__()
self.engine = engine(
 include_top=False, input_shape=input_shape, weights=weights)
 self.avg_pool2d = tf.keras.layers.GlobalAveragePooling2D()
self.dropout = tf.keras.layers.Dropout(0.5)
 self.dense_1 = tf.keras.layers.Dense(1024)
 self.dense_2 = tf.keras.layers.Dense(5,activation='sigmoid')
 @tf.function
def call(self, inputs, **kwargs):
x = tf.reshape(inputs,(-1, IMG_SIZE, IMG_SIZE, 3))
x = self.engine(x)
 shape = x.shape
x = tf.reshape(x,(-1, N_TILES, shape[1], shape[2], shape[3]))
x = tf.transpose(x, perm=[0, 2, 1, 3, 4])
 x = tf.reshape(x,(-1, shape[1], N_TILES*shape[2], shape[3]))
x = self.avg_pool2d(x)
 x = self.dropout(x, training=False)
 x = self.dense_1(x)
 x = tf.nn.relu(x)
 return self.dense_2(x)
 is_ef = True
backbone_name = 'efficientnet-b0'
N_TILES = 48
IMG_SIZE = 256
if backbone_name.startswith('efficientnet'):
model_fn = getattr(efn, f'EfficientNetB{backbone_name[-1]}')
 model = ConvNet(engine=model_fn, input_shape=(IMG_SIZE, IMG_SIZE, 3), weights=None)
 TRAIN = '.. /input/prostate-cancer-grade-assessment/train_images/'
MASKS = '.. /input/prostate-cancer-grade-assessment/train_label_masks/'
BASE_PATH = '.. /input/prostate-cancer-grade-assessment/'
train = pd.read_csv(BASE_PATH + "train.csv")
 train.head()
sub = pd.read_csv(".. /input/prostate-cancer-grade-assessment/sample_submission.csv")
 sub.head()
test = pd.read_csv(".. /input/prostate-cancer-grade-assessment/test.csv")
 test.head()
TEST = '.. /input/prostate-cancer-grade-assessment/test_images/'
PRED_PATH = TEST
df = sub
t_df = test
transforms_val0 = albumentations.Compose([])
 transforms_val1 = albumentations.Compose([
albumentations.VerticalFlip(p=1)
 ])
 transforms_val2= albumentations.Compose([
albumentations.HorizontalFlip(p=0.5),
albumentations.VerticalFlip(p=0.5),
])
 transforms_val3= albumentations.Compose([
albumentations.HorizontalFlip(p=0.5),
albumentations.VerticalFlip(p=0.5),
])
 n_TTA = 2
dummy_data = tf.zeros(( n_TTA * N_TILES, IMG_SIZE, IMG_SIZE, 3), dtype=tf.float32)
 _ = model(dummy_data)
if not RECALCULATE_PEAK_FEATURES:
test_peaks = pd.read_pickle(PREPROCESSED_DIR + '/test_peaks.pkl')
 test_peaks.to_pickle("test_peaks.pkl" )<merge>
model.compile(optimizer = tf.keras.optimizers.Adam(lr= 1e-05), loss= tf.nn.sigmoid_cross_entropy_with_logits)
 model.load_weights('.. /input/pandaenetb042x256x256x3/efficientnet-b0-48-full-epochs60.h5')
 if os.path.exists(PRED_PATH):
predictions10 = []
for index, row in tqdm(df.iterrows() , total = df.shape[0]):
image_id = row['image_id']
img_path = PRED_PATH + image_id + '.tiff'
img = skimage.io.MultiImage(img_path)[1]
patches = tile(img)
 patches3 = patches.copy()
patches4 = patches.copy()
k = 0
while k < N_TILES:
patches3[k, ] = transforms_val2(image=patches3[k, ])['image']
patches4[k, ] = transforms_val3(image=patches4[k, ])['image']
k += 1
image = np.stack([patches3, patches4])
 image = image / 255.0
pred = model.predict(image)
 isup = 0.5*np.sum(pred)
 predictions10.append(isup)
 del patches, img
gc.collect()
else:
PRED_PATH = TRAIN
df_train = pd.read_csv(os.path.join(data_dir, 'train.csv'))
df = df_train.loc[:10]
df = df[['image_id','isup_grade']].copy()
predictions10 = []
for index, row in tqdm(df.iterrows() , total = df.shape[0]):
image_id = row['image_id']
img_path = PRED_PATH + image_id + '.tiff'
img = skimage.io.MultiImage(img_path)[1]
patches = tile(img)
 patches3 = patches.copy()
patches4 = patches.copy()
k = 0
while k < N_TILES:
patches3[k, ] = transforms_val2(image=patches3[k, ])['image']
patches4[k, ] = transforms_val3(image=patches4[k, ])['image']
k += 1
image = np.stack([patches3, patches4])
 image = image / 255.0
pred = model.predict(image)
 isup = 0.5*np.sum(pred)
 predictions10.append(isup)
 del patches, img
gc.collect()
if os.path.exists(PRED_PATH):
predictions12 = []
for index, row in tqdm(df.iterrows() , total = df.shape[0]):
image_id = row['image_id']
img_path = PRED_PATH + image_id + '.tiff'
img = skimage.io.MultiImage(img_path)[1]
patches = tile2(img)
 patches3 = patches.copy()
patches4 = patches.copy()
k = 0
while k < N_TILES:
patches3[k, ] = transforms_val2(image=patches3[k, ])['image']
patches4[k, ] = transforms_val3(image=patches4[k, ])['image']
k += 1
image = np.stack([patches3, patches4])
 image = image / 255.0
pred = model.predict(image)
 isup = 0.5*np.sum(pred)
 predictions12.append(isup)
 del patches, img
gc.collect()
else:
PRED_PATH = TRAIN
df_train = pd.read_csv(os.path.join(data_dir, 'train.csv'))
df = df_train.loc[:10]
df = df[['image_id','isup_grade']].copy()
predictions12 = []
for index, row in tqdm(df.iterrows() , total = df.shape[0]):
image_id = row['image_id']
img_path = PRED_PATH + image_id + '.tiff'
img = skimage.io.MultiImage(img_path)[1]
patches = tile2(img)
 patches3 = patches.copy()
patches4 = patches.copy()
k = 0
while k < N_TILES:
patches3[k, ] = transforms_val2(image=patches3[k, ])['image']
patches4[k, ] = transforms_val3(image=patches4[k, ])['image']
k += 1
image = np.stack([patches3, patches4])
 image = image / 255.0
pred = model.predict(image)
 isup = 0.5*np.sum(pred)
 predictions12.append(isup)
 del patches, img
gc.collect()
test_peaks = pd.merge(test_peaks, meta_test_df[['signal_id', 'id_measurement']], on='signal_id', how='left' )<feature_engineering>
class ConvNet(tf.keras.Model):
def __init__(self, engine, input_shape, weights):
super(ConvNet, self ).__init__()
self.engine = engine(
 include_top=False, input_shape=input_shape, weights=weights)
 self.avg_pool2d = tf.keras.layers.GlobalAveragePooling2D()
self.dropout = tf.keras.layers.Dropout(0.5)
 self.dense_1 = tf.keras.layers.Dense(1024)
 self.dense_2 = tf.keras.layers.Dense(5,activation='sigmoid')
 @tf.function
def call(self, inputs, **kwargs):
x = tf.reshape(inputs,(-1, IMG_SIZE, IMG_SIZE, 3))
x = self.engine(x)
 shape = x.shape
x = tf.reshape(x,(-1, N_TILES, shape[1], shape[2], shape[3]))
x = tf.transpose(x, perm=[0, 2, 1, 3, 4])
 x = tf.reshape(x,(-1, shape[1], N_TILES*shape[2], shape[3]))
x = self.avg_pool2d(x)
 x = self.dropout(x, training=False)
 x = self.dense_1(x)
 x = tf.nn.relu(x)
 return self.dense_2(x)
 is_ef = True
backbone_name = 'efficientnet-b1'
N_TILES = 48
IMG_SIZE = 256
if backbone_name.startswith('efficientnet'):
model_fn = getattr(efn, f'EfficientNetB{backbone_name[-1]}')
 model = ConvNet(engine=model_fn, input_shape=(IMG_SIZE, IMG_SIZE, 3), weights=None)
 n_TTA = 2
dummy_data = tf.zeros(( n_TTA * N_TILES, IMG_SIZE, IMG_SIZE, 3), dtype=tf.float32)
 _ = model(dummy_data)
 TRAIN = '.. /input/prostate-cancer-grade-assessment/train_images/'
MASKS = '.. /input/prostate-cancer-grade-assessment/train_label_masks/'
BASE_PATH = '.. /input/prostate-cancer-grade-assessment/'
train = pd.read_csv(BASE_PATH + "train.csv")
 train.head()
sub = pd.read_csv(".. /input/prostate-cancer-grade-assessment/sample_submission.csv")
 sub.head()
test = pd.read_csv(".. /input/prostate-cancer-grade-assessment/test.csv")
 test.head()
TEST = '.. /input/prostate-cancer-grade-assessment/test_images/'
PRED_PATH = TEST
df = sub
t_df = test
def create_features(
 meta_df,
peaks_df, 
):
res = process(peaks_df, meta_df)
 return res<feature_engineering>
model.compile(optimizer = tf.keras.optimizers.Adam(lr= 1e-05), loss= tf.nn.sigmoid_cross_entropy_with_logits)
 model.load_weights('.. /input/pandaenetb042x256x256x3/efficientnet-b1-48-full-epochs60.h5')
 if os.path.exists(PRED_PATH):
predictions20 = []
for index, row in tqdm(df.iterrows() , total = df.shape[0]):
image_id = row['image_id']
img_path = PRED_PATH + image_id + '.tiff'
img = skimage.io.MultiImage(img_path)[1]
patches = tile(img)
 patches3 = patches.copy()
patches4 = patches.copy()
k = 0
while k < N_TILES:
patches3[k, ] = transforms_val2(image=patches3[k, ])['image']
patches4[k, ] = transforms_val3(image=patches4[k, ])['image']
k += 1
image = np.stack([patches3, patches4])
 image = image / 255.0
pred = model.predict(image)
 isup = 0.5*np.sum(pred)
 predictions20.append(isup)
 del patches, img
gc.collect()
else:
PRED_PATH = TRAIN
df_train = pd.read_csv(os.path.join(data_dir, 'train.csv'))
df = df_train.loc[:10]
df = df[['image_id','isup_grade']].copy()
predictions20 = []
for index, row in tqdm(df.iterrows() , total = df.shape[0]):
image_id = row['image_id']
img_path = PRED_PATH + image_id + '.tiff'
img = skimage.io.MultiImage(img_path)[1]
patches = tile(img)
 patches3 = patches.copy()
patches4 = patches.copy()
k = 0
while k < N_TILES:
patches3[k, ] = transforms_val2(image=patches3[k, ])['image']
patches4[k, ] = transforms_val3(image=patches4[k, ])['image']
k += 1
image = np.stack([patches3, patches4])
 image = image / 255.0
pred = model.predict(image)
 isup = 0.5*np.sum(pred)
 predictions20.append(isup)
 del patches, img
gc.collect()
if os.path.exists(PRED_PATH):
predictions22 = []
for index, row in tqdm(df.iterrows() , total = df.shape[0]):
image_id = row['image_id']
img_path = PRED_PATH + image_id + '.tiff'
img = skimage.io.MultiImage(img_path)[1]
patches = tile2(img)
 patches3 = patches.copy()
patches4 = patches.copy()
k = 0
while k < N_TILES:
patches3[k, ] = transforms_val2(image=patches3[k, ])['image']
patches4[k, ] = transforms_val3(image=patches4[k, ])['image']
k += 1
image = np.stack([patches3, patches4])
 image = image / 255.0
pred = model.predict(image)
 isup = 0.5*np.sum(pred)
 predictions22.append(isup)
 del patches, img
gc.collect()
else:
PRED_PATH = TRAIN
df_train = pd.read_csv(os.path.join(data_dir, 'train.csv'))
df = df_train.loc[:10]
df = df[['image_id','isup_grade']].copy()
predictions22 = []
for index, row in tqdm(df.iterrows() , total = df.shape[0]):
image_id = row['image_id']
img_path = PRED_PATH + image_id + '.tiff'
img = skimage.io.MultiImage(img_path)[1]
patches = tile2(img)
 patches3 = patches.copy()
patches4 = patches.copy()
k = 0
while k < N_TILES:
patches3[k, ] = transforms_val2(image=patches3[k, ])['image']
patches4[k, ] = transforms_val3(image=patches4[k, ])['image']
k += 1
image = np.stack([patches3, patches4])
 image = image / 255.0
pred = model.predict(image)
 isup = 0.5*np.sum(pred)
 predictions22.append(isup)
 del patches, img
gc.collect()
%%time
X_train_full = create_features(
 meta_train_df,
train_peaks,
)<data_type_conversions>
class ConvNet(tf.keras.Model):
def __init__(self, engine, input_shape, weights):
super(ConvNet, self ).__init__()
self.engine = engine(
 include_top=False, input_shape=input_shape, weights=weights)
 self.avg_pool2d = tf.keras.layers.GlobalAveragePooling2D()
self.dropout = tf.keras.layers.Dropout(0.5)
 self.dense_1 = tf.keras.layers.Dense(1024)
 self.dense_2 = tf.keras.layers.Dense(5,activation='sigmoid')
 @tf.function
def call(self, inputs, **kwargs):
x = tf.reshape(inputs,(-1, IMG_SIZE, IMG_SIZE, 3))
x = self.engine(x)
 shape = x.shape
x = tf.reshape(x,(-1, N_TILES, shape[1], shape[2], shape[3]))
x = tf.transpose(x, perm=[0, 2, 1, 3, 4])
 x = tf.reshape(x,(-1, shape[1], N_TILES*shape[2], shape[3]))
x = self.avg_pool2d(x)
 x = self.dropout(x, training=False)
 x = self.dense_1(x)
 x = tf.nn.relu(x)
 return self.dense_2(x)
 is_ef = True
backbone_name = 'efficientnet-b2'
N_TILES = 48
IMG_SIZE = 256
if backbone_name.startswith('efficientnet'):
model_fn = getattr(efn, f'EfficientNetB{backbone_name[-1]}')
 model = ConvNet(engine=model_fn, input_shape=(IMG_SIZE, IMG_SIZE, 3), weights=None)
 n_TTA = 2
dummy_data = tf.zeros(( n_TTA * N_TILES, IMG_SIZE, IMG_SIZE, 3), dtype=tf.float32)
 _ = model(dummy_data)
 TRAIN = '.. /input/prostate-cancer-grade-assessment/train_images/'
MASKS = '.. /input/prostate-cancer-grade-assessment/train_label_masks/'
BASE_PATH = '.. /input/prostate-cancer-grade-assessment/'
train = pd.read_csv(BASE_PATH + "train.csv")
 train.head()
sub = pd.read_csv(".. /input/prostate-cancer-grade-assessment/sample_submission.csv")
 sub.head()
test = pd.read_csv(".. /input/prostate-cancer-grade-assessment/test.csv")
 test.head()
TEST = '.. /input/prostate-cancer-grade-assessment/test_images/'
PRED_PATH = TEST
df = sub
t_df = test
y_train =(
 meta_train_df.groupby('id_measurement')['target'].sum().round(0 ).astype(np.int)
 != 0
).astype(np.float)
 assert np.all(y_train.index.values == X_train_full.index.values )<feature_engineering>
model.compile(optimizer = tf.keras.optimizers.Adam(lr= 1e-05), loss= tf.nn.sigmoid_cross_entropy_with_logits)
 model.load_weights('.. /input/pandaenetb042x256x256x3/efficientnet-b2-48-full-epochs60.h5')
 if os.path.exists(PRED_PATH):
predictions30 = []
for index, row in tqdm(df.iterrows() , total = df.shape[0]):
image_id = row['image_id']
img_path = PRED_PATH + image_id + '.tiff'
img = skimage.io.MultiImage(img_path)[1]
patches = tile(img)
 patches3 = patches.copy()
patches4 = patches.copy()
k = 0
while k < N_TILES:
patches3[k, ] = transforms_val2(image=patches3[k, ])['image']
patches4[k, ] = transforms_val3(image=patches4[k, ])['image']
k += 1
image = np.stack([patches3, patches4])
 image = image / 255.0
pred = model.predict(image)
 isup = 0.5*np.sum(pred)
 predictions30.append(isup)
 del patches, img
gc.collect()
else:
PRED_PATH = TRAIN
df_train = pd.read_csv(os.path.join(data_dir, 'train.csv'))
df = df_train.loc[:10]
df = df[['image_id','isup_grade']].copy()
predictions30 = []
for index, row in tqdm(df.iterrows() , total = df.shape[0]):
image_id = row['image_id']
img_path = PRED_PATH + image_id + '.tiff'
img = skimage.io.MultiImage(img_path)[1]
patches = tile(img)
 patches3 = patches.copy()
patches4 = patches.copy()
k = 0
while k < N_TILES:
patches3[k, ] = transforms_val2(image=patches3[k, ])['image']
patches4[k, ] = transforms_val3(image=patches4[k, ])['image']
k += 1
image = np.stack([patches3, patches4])
 image = image / 255.0
pred = model.predict(image)
 isup = 0.5*np.sum(pred)
 predictions30.append(isup)
 del patches, img
gc.collect()
if os.path.exists(PRED_PATH):
predictions32 = []
for index, row in tqdm(df.iterrows() , total = df.shape[0]):
image_id = row['image_id']
img_path = PRED_PATH + image_id + '.tiff'
img = skimage.io.MultiImage(img_path)[1]
patches = tile2(img)
 patches3 = patches.copy()
patches4 = patches.copy()
k = 0
while k < N_TILES:
patches3[k, ] = transforms_val2(image=patches3[k, ])['image']
patches4[k, ] = transforms_val3(image=patches4[k, ])['image']
k += 1
image = np.stack([patches3, patches4])
 image = image / 255.0
pred = model.predict(image)
 isup = 0.5*np.sum(pred)
 predictions32.append(isup)
 del patches, img
gc.collect()
else:
PRED_PATH = TRAIN
df_train = pd.read_csv(os.path.join(data_dir, 'train.csv'))
df = df_train.loc[:10]
df = df[['image_id','isup_grade']].copy()
predictions32 = []
for index, row in tqdm(df.iterrows() , total = df.shape[0]):
image_id = row['image_id']
img_path = PRED_PATH + image_id + '.tiff'
img = skimage.io.MultiImage(img_path)[1]
patches = tile2(img)
 patches3 = patches.copy()
patches4 = patches.copy()
k = 0
while k < N_TILES:
patches3[k, ] = transforms_val2(image=patches3[k, ])['image']
patches4[k, ] = transforms_val3(image=patches4[k, ])['image']
k += 1
image = np.stack([patches3, patches4])
 image = image / 255.0
pred = model.predict(image)
 isup = 0.5*np.sum(pred)
 predictions32.append(isup)
 del patches, img
gc.collect()
%%time
X_test_full = create_features(
 meta_test_df,
test_peaks,
)<define_variables>
class ConvNet(tf.keras.Model):
def __init__(self, engine, input_shape, weights):
super(ConvNet, self ).__init__()
self.engine = engine(
 include_top=False, input_shape=input_shape, weights=weights)
 self.avg_pool2d = tf.keras.layers.GlobalAveragePooling2D()
self.dropout = tf.keras.layers.Dropout(0.5)
 self.dense_1 = tf.keras.layers.Dense(1024)
 self.dense_2 = tf.keras.layers.Dense(5,activation='sigmoid')
 @tf.function
def call(self, inputs, **kwargs):
x = tf.reshape(inputs,(-1, IMG_SIZE, IMG_SIZE, 3))
x = self.engine(x)
 shape = x.shape
x = tf.reshape(x,(-1, N_TILES, shape[1], shape[2], shape[3]))
x = tf.transpose(x, perm=[0, 2, 1, 3, 4])
 x = tf.reshape(x,(-1, shape[1], N_TILES*shape[2], shape[3]))
x = self.avg_pool2d(x)
 x = self.dropout(x, training=False)
 x = self.dense_1(x)
 x = tf.nn.relu(x)
 return self.dense_2(x)
 N_TILES = 48
IMG_SIZE = 256
model_fn = getattr(den, 'DenseNet121')
 model = ConvNet(engine=model_fn, input_shape=(IMG_SIZE, IMG_SIZE, 3), weights=None)
 n_TTA = 4
dummy_data = tf.zeros(( n_TTA * N_TILES, IMG_SIZE, IMG_SIZE, 3), dtype=tf.float32)
 _ = model(dummy_data)
 TRAIN = '.. /input/prostate-cancer-grade-assessment/train_images/'
MASKS = '.. /input/prostate-cancer-grade-assessment/train_label_masks/'
BASE_PATH = '.. /input/prostate-cancer-grade-assessment/'
train = pd.read_csv(BASE_PATH + "train.csv")
 train.head()
sub = pd.read_csv(".. /input/prostate-cancer-grade-assessment/sample_submission.csv")
 sub.head()
test = pd.read_csv(".. /input/prostate-cancer-grade-assessment/test.csv")
 test.head()
TEST = '.. /input/prostate-cancer-grade-assessment/test_images/'
PRED_PATH = TEST
df = sub
t_df = test
default_features_to_drop = [
'id_measurement',
'phase',
'fft_phase',
'signal_id',
]<define_variables>
model.compile(optimizer = tf.keras.optimizers.Adam(lr= 1e-05), loss= tf.nn.sigmoid_cross_entropy_with_logits)
 model.load_weights('.. /input/pandaenetb042x256x256x3/DenseNet121-48-full-epochs60.h5')
 if os.path.exists(PRED_PATH):
predictions40 = []
for index, row in tqdm(df.iterrows() , total = df.shape[0]):
image_id = row['image_id']
img_path = PRED_PATH + image_id + '.tiff'
img = skimage.io.MultiImage(img_path)[1]
patches = tile(img)
 patches1 = patches.copy()
patches2 = patches.copy()
patches3 = patches.copy()
patches4 = patches.copy()
k = 0
while k < N_TILES:
patches1[k, ] = transforms_val0(image=patches1[k, ])['image']
patches2[k, ] = transforms_val1(image=patches2[k, ])['image']
patches3[k, ] = transforms_val2(image=patches3[k, ])['image']
patches4[k, ] = transforms_val3(image=patches4[k, ])['image']
k += 1
image = np.stack([patches1, patches2, patches3, patches4])
 image = image / 255.0
pred = model.predict(image)
 isup = 0.25*np.sum(pred)
 predictions40.append(isup)
 del patches, img
gc.collect()
else:
PRED_PATH = TRAIN
df_train = pd.read_csv(os.path.join(data_dir, 'train.csv'))
df = df_train.loc[:10]
df = df[['image_id','isup_grade']].copy()
predictions40 = []
for index, row in tqdm(df.iterrows() , total = df.shape[0]):
image_id = row['image_id']
img_path = PRED_PATH + image_id + '.tiff'
img = skimage.io.MultiImage(img_path)[1]
patches = tile(img)
 patches1 = patches.copy()
patches2 = patches.copy()
patches3 = patches.copy()
patches4 = patches.copy()
k = 0
while k < N_TILES:
patches1[k, ] = transforms_val0(image=patches1[k, ])['image']
patches2[k, ] = transforms_val1(image=patches2[k, ])['image']
patches3[k, ] = transforms_val2(image=patches3[k, ])['image']
patches4[k, ] = transforms_val3(image=patches4[k, ])['image']
k += 1
image = np.stack([patches1, patches2, patches3, patches4])
 image = image / 255.0
pred = model.predict(image)
 isup = 0.25*np.sum(pred)
 predictions40.append(isup)
 del patches, img
gc.collect()
if os.path.exists(PRED_PATH):
predictions42 = []
for index, row in tqdm(df.iterrows() , total = df.shape[0]):
image_id = row['image_id']
img_path = PRED_PATH + image_id + '.tiff'
img = skimage.io.MultiImage(img_path)[1]
patches = tile2(img)
 patches1 = patches.copy()
patches2 = patches.copy()
patches3 = patches.copy()
patches4 = patches.copy()
k = 0
while k < N_TILES:
patches1[k, ] = transforms_val0(image=patches1[k, ])['image']
patches2[k, ] = transforms_val1(image=patches2[k, ])['image']
patches3[k, ] = transforms_val2(image=patches3[k, ])['image']
patches4[k, ] = transforms_val3(image=patches4[k, ])['image']
k += 1
image = np.stack([patches1, patches2, patches3, patches4])
 image = image / 255.0
pred = model.predict(image)
 isup = 0.25*np.sum(pred)
 predictions42.append(isup)
 del patches, img
gc.collect()
else:
PRED_PATH = TRAIN
df_train = pd.read_csv(os.path.join(data_dir, 'train.csv'))
df = df_train.loc[:10]
df = df[['image_id','isup_grade']].copy()
predictions42 = []
for index, row in tqdm(df.iterrows() , total = df.shape[0]):
image_id = row['image_id']
img_path = PRED_PATH + image_id + '.tiff'
img = skimage.io.MultiImage(img_path)[1]
patches = tile2(img)
 patches1 = patches.copy()
patches2 = patches.copy()
patches3 = patches.copy()
patches4 = patches.copy()
k = 0
while k < N_TILES:
patches1[k, ] = transforms_val0(image=patches1[k, ])['image']
patches2[k, ] = transforms_val1(image=patches2[k, ])['image']
patches3[k, ] = transforms_val2(image=patches3[k, ])['image']
patches4[k, ] = transforms_val3(image=patches4[k, ])['image']
k += 1
image = np.stack([patches1, patches2, patches3, patches4])
 image = image / 255.0
pred = model.predict(image)
 isup = 0.25*np.sum(pred)
 predictions42.append(isup)
 del patches, img
gc.collect()
del model, dummy_data, sub, pred, train, isup, image
del patches1,patches2,patches3,patches4
gc.collect()
if USE_SIMPLIFIED_VERSION:
additional_features_to_drop = [
'abs_small_dist_to_min_mean_Q02',
'height_mean_Q02',
'ratio_next_mean_Q02',
'ratio_prev_mean_Q02',
]
else:
additional_features_to_drop = []<define_variables>
sz = 256
N = 42
def tile(img):
result = []
shape = img.shape
pad0,pad1 =(sz - shape[0]%sz)%sz,(sz - shape[1]%sz)%sz
img = np.pad(img,[[pad0//2,pad0-pad0//2],[pad1//2,pad1-pad1//2],[0,0]],constant_values=255)
 img = img.reshape(img.shape[0]//sz,sz,img.shape[1]//sz,sz,3)
 img = img.transpose(0,2,1,3,4 ).reshape(-1,sz,sz,3)
 if len(img)< N:
img = np.pad(img,[[0,N-len(img)],[0,0],[0,0],[0,0]],constant_values=255)
 idxs = np.argsort(img.reshape(img.shape[0],-1 ).sum(-1)) [:N]
img = img[idxs]
return img
def tile2(img):
result = []
shape = img.shape
pad0,pad1 =(sz - shape[0]%sz)%sz +(( sz * 2)// 2),(sz - shape[1]%sz)%sz +(( sz * 2)// 2)
 img = np.pad(img,[[pad0//2,pad0-pad0//2],[pad1//2,pad1-pad1//2],[0,0]],constant_values=255)
 img = img.reshape(img.shape[0]//sz,sz,img.shape[1]//sz,sz,3)
 img = img.transpose(0,2,1,3,4 ).reshape(-1,sz,sz,3)
 if len(img)< N:
img = np.pad(img,[[0,N-len(img)],[0,0],[0,0],[0,0]],constant_values=255)
 idxs = np.argsort(img.reshape(img.shape[0],-1 ).sum(-1)) [:N]
img = img[idxs]
return img
class ConvNet(tf.keras.Model):
def __init__(self, engine, input_shape, weights):
super(ConvNet, self ).__init__()
self.engine = engine(
 include_top=False, input_shape=input_shape, weights=weights)
 self.avg_pool2d = tf.keras.layers.GlobalAveragePooling2D()
self.dropout = tf.keras.layers.Dropout(0.5)
 self.dense_1 = tf.keras.layers.Dense(1024)
 self.dense_2 = tf.keras.layers.Dense(1)
 @tf.function
def call(self, inputs, **kwargs):
x = tf.reshape(inputs,(-1, IMG_SIZE, IMG_SIZE, 3))
x = self.engine(x)
 shape = x.shape
x = tf.reshape(x,(-1, N_TILES, shape[1], shape[2], shape[3]))
x = tf.transpose(x, perm=[0, 2, 1, 3, 4])
 x = tf.reshape(x,(-1, shape[1], N_TILES*shape[2], shape[3]))
x = self.avg_pool2d(x)
 x = self.dropout(x, training=False)
 x = self.dense_1(x)
 x = tf.nn.relu(x)
 return self.dense_2(x)
 is_ef = True
backbone_name = 'efficientnet-b0'
N_TILES = 42
IMG_SIZE = 256
if backbone_name.startswith('efficientnet'):
model_fn = getattr(efn, f'EfficientNetB{backbone_name[-1]}')
 model = ConvNet(engine=model_fn, input_shape=(IMG_SIZE, IMG_SIZE, 3), weights=None)
 TRAIN = '.. /input/prostate-cancer-grade-assessment/train_images/'
MASKS = '.. /input/prostate-cancer-grade-assessment/train_label_masks/'
BASE_PATH = '.. /input/prostate-cancer-grade-assessment/'
train = pd.read_csv(BASE_PATH + "train.csv")
 train.head()
sub = pd.read_csv(".. /input/prostate-cancer-grade-assessment/sample_submission.csv")
 sub.head()
test = pd.read_csv(".. /input/prostate-cancer-grade-assessment/test.csv")
 test.head()
TEST = '.. /input/prostate-cancer-grade-assessment/test_images/'
PRED_PATH = TEST
df = sub
t_df = test
n_TTA = 2
dummy_data = tf.zeros(( n_TTA * N_TILES, IMG_SIZE, IMG_SIZE, 3), dtype=tf.float32)
 _ = model(dummy_data)
features_to_drop =(
 default_features_to_drop
+ additional_features_to_drop
)<define_variables>
model.compile(optimizer = tf.keras.optimizers.Adam(lr= 1e-05), loss= tf.nn.sigmoid_cross_entropy_with_logits)
 model.load_weights('.. /input/pandaenetb042x256x256x3/efficientnet-b0-fold0-epochs40.h5')
 if os.path.exists(PRED_PATH):
predictions50 = []
for index, row in tqdm(df.iterrows() , total = df.shape[0]):
image_id = row['image_id']
img_path = PRED_PATH + image_id + '.tiff'
img = skimage.io.MultiImage(img_path)[1]
patches = tile(img)
 patches3 = patches.copy()
patches4 = patches.copy()
k = 0
while k < N_TILES:
patches3[k, ] = transforms_val2(image=patches3[k, ])['image']
patches4[k, ] = transforms_val3(image=patches4[k, ])['image']
k += 1
image = np.stack([patches3, patches4])
 image = image / 255.0
pred = model.predict(image)
 isup = np.mean(pred)
 predictions50.append(isup)
 del patches, img
gc.collect()
else:
PRED_PATH = TRAIN
df_train = pd.read_csv(os.path.join(data_dir, 'train.csv'))
df = df_train.loc[:10]
df = df[['image_id','isup_grade']].copy()
predictions50 = []
for index, row in tqdm(df.iterrows() , total = df.shape[0]):
image_id = row['image_id']
img_path = PRED_PATH + image_id + '.tiff'
img = skimage.io.MultiImage(img_path)[1]
patches = tile(img)
 patches3 = patches.copy()
patches4 = patches.copy()
k = 0
while k < N_TILES:
patches3[k, ] = transforms_val2(image=patches3[k, ])['image']
patches4[k, ] = transforms_val3(image=patches4[k, ])['image']
k += 1
image = np.stack([patches3, patches4])
 image = image / 255.0
pred = model.predict(image)
 isup = np.mean(pred)
 predictions50.append(isup)
 del patches, img
gc.collect()
if os.path.exists(PRED_PATH):
predictions52 = []
for index, row in tqdm(df.iterrows() , total = df.shape[0]):
image_id = row['image_id']
img_path = PRED_PATH + image_id + '.tiff'
img = skimage.io.MultiImage(img_path)[1]
patches = tile2(img)
 patches3 = patches.copy()
patches4 = patches.copy()
k = 0
while k < N_TILES:
patches3[k, ] = transforms_val2(image=patches3[k, ])['image']
patches4[k, ] = transforms_val3(image=patches4[k, ])['image']
k += 1
image = np.stack([patches3, patches4])
 image = image / 255.0
pred = model.predict(image)
 isup = np.mean(pred)
 predictions52.append(isup)
 del patches, img
gc.collect()
else:
PRED_PATH = TRAIN
df_train = pd.read_csv(os.path.join(data_dir, 'train.csv'))
df = df_train.loc[:10]
df = df[['image_id','isup_grade']].copy()
predictions52 = []
for index, row in tqdm(df.iterrows() , total = df.shape[0]):
image_id = row['image_id']
img_path = PRED_PATH + image_id + '.tiff'
img = skimage.io.MultiImage(img_path)[1]
patches = tile2(img)
 patches3 = patches.copy()
patches4 = patches.copy()
k = 0
while k < N_TILES:
patches3[k, ] = transforms_val2(image=patches3[k, ])['image']
patches4[k, ] = transforms_val3(image=patches4[k, ])['image']
k += 1
image = np.stack([patches3, patches4])
 image = image / 255.0
pred = model.predict(image)
 isup = np.mean(pred)
 predictions52.append(isup)
 del patches, img
gc.collect()
if USE_SIMPLIFIED_VERSION:
feature_names = sorted([c for c in X_train_full.columns if c not in features_to_drop])
 else:
feature_names = [
'peak_count_Q13',
'abs_small_dist_to_min_mean_Q02',
'height_mean_Q02',
'height_std_Q02',
'sawtooth_rmse_mean_Q02',
'peak_count_Q02',
'ratio_next_mean_Q02',
'ratio_prev_mean_Q02',
'peak_count_total'
]<prepare_x_and_y>
model.compile(optimizer = tf.keras.optimizers.Adam(lr= 1e-05), loss= tf.nn.sigmoid_cross_entropy_with_logits)
 model.load_weights('.. /input/pandaenetb042x256x256x3/efficientnet-b0-fold4-epochs60.h5')
 if os.path.exists(PRED_PATH):
predictions60 = []
for index, row in tqdm(df.iterrows() , total = df.shape[0]):
image_id = row['image_id']
img_path = PRED_PATH + image_id + '.tiff'
img = skimage.io.MultiImage(img_path)[1]
patches = tile(img)
 patches3 = patches.copy()
patches4 = patches.copy()
k = 0
while k < N_TILES:
patches3[k, ] = transforms_val2(image=patches3[k, ])['image']
patches4[k, ] = transforms_val3(image=patches4[k, ])['image']
k += 1
image = np.stack([patches3, patches4])
 image = image / 255.0
pred = model.predict(image)
 isup = np.mean(pred)
 predictions60.append(isup)
 del patches, img
gc.collect()
else:
PRED_PATH = TRAIN
df_train = pd.read_csv(os.path.join(data_dir, 'train.csv'))
df = df_train.loc[:10]
df = df[['image_id','isup_grade']].copy()
predictions60 = []
for index, row in tqdm(df.iterrows() , total = df.shape[0]):
image_id = row['image_id']
img_path = PRED_PATH + image_id + '.tiff'
img = skimage.io.MultiImage(img_path)[1]
patches = tile(img)
 patches3 = patches.copy()
patches4 = patches.copy()
k = 0
while k < N_TILES:
patches3[k, ] = transforms_val2(image=patches3[k, ])['image']
patches4[k, ] = transforms_val3(image=patches4[k, ])['image']
k += 1
image = np.stack([patches3, patches4])
 image = image / 255.0
pred = model.predict(image)
 isup = np.mean(pred)
 predictions60.append(isup)
 del patches, img
gc.collect()
if os.path.exists(PRED_PATH):
predictions62 = []
for index, row in tqdm(df.iterrows() , total = df.shape[0]):
image_id = row['image_id']
img_path = PRED_PATH + image_id + '.tiff'
img = skimage.io.MultiImage(img_path)[1]
patches = tile2(img)
 patches3 = patches.copy()
patches4 = patches.copy()
k = 0
while k < N_TILES:
patches3[k, ] = transforms_val2(image=patches3[k, ])['image']
patches4[k, ] = transforms_val3(image=patches4[k, ])['image']
k += 1
image = np.stack([patches3, patches4])
 image = image / 255.0
pred = model.predict(image)
 isup = np.mean(pred)
 predictions62.append(isup)
 del patches, img
gc.collect()
else:
PRED_PATH = TRAIN
df_train = pd.read_csv(os.path.join(data_dir, 'train.csv'))
df = df_train.loc[:10]
df = df[['image_id','isup_grade']].copy()
predictions62 = []
for index, row in tqdm(df.iterrows() , total = df.shape[0]):
image_id = row['image_id']
img_path = PRED_PATH + image_id + '.tiff'
img = skimage.io.MultiImage(img_path)[1]
patches = tile2(img)
 patches3 = patches.copy()
patches4 = patches.copy()
k = 0
while k < N_TILES:
patches3[k, ] = transforms_val2(image=patches3[k, ])['image']
patches4[k, ] = transforms_val3(image=patches4[k, ])['image']
k += 1
image = np.stack([patches3, patches4])
 image = image / 255.0
pred = model.predict(image)
 isup = np.mean(pred)
 predictions62.append(isup)
 del patches, img
gc.collect()
X_train = X_train_full[feature_names]
X_test = X_test_full[feature_names]<compute_test_metric>
model.compile(optimizer = tf.keras.optimizers.Adam(lr= 1e-05), loss= tf.nn.sigmoid_cross_entropy_with_logits)
 model.load_weights('.. /input/pandaenetb042x256x256x3/efficientnet-b0-fold2-epochs40.h5')
 if os.path.exists(PRED_PATH):
predictions70 = []
for index, row in tqdm(df.iterrows() , total = df.shape[0]):
image_id = row['image_id']
img_path = PRED_PATH + image_id + '.tiff'
img = skimage.io.MultiImage(img_path)[1]
patches = tile(img)
 patches3 = patches.copy()
patches4 = patches.copy()
k = 0
while k < N_TILES:
patches3[k, ] = transforms_val2(image=patches3[k, ])['image']
patches4[k, ] = transforms_val3(image=patches4[k, ])['image']
k += 1
image = np.stack([patches3, patches4])
 image = image / 255.0
pred = model.predict(image)
 isup = np.mean(pred)
 predictions70.append(isup)
 del patches, img
gc.collect()
else:
PRED_PATH = TRAIN
df_train = pd.read_csv(os.path.join(data_dir, 'train.csv'))
df = df_train.loc[:10]
df = df[['image_id','isup_grade']].copy()
predictions70 = []
for index, row in tqdm(df.iterrows() , total = df.shape[0]):
image_id = row['image_id']
img_path = PRED_PATH + image_id + '.tiff'
img = skimage.io.MultiImage(img_path)[1]
patches = tile(img)
 patches3 = patches.copy()
patches4 = patches.copy()
k = 0
while k < N_TILES:
patches3[k, ] = transforms_val2(image=patches3[k, ])['image']
patches4[k, ] = transforms_val3(image=patches4[k, ])['image']
k += 1
image = np.stack([patches3, patches4])
 image = image / 255.0
pred = model.predict(image)
 isup = np.mean(pred)
 predictions70.append(isup)
 del patches, img
gc.collect()
if os.path.exists(PRED_PATH):
predictions72 = []
for index, row in tqdm(df.iterrows() , total = df.shape[0]):
image_id = row['image_id']
img_path = PRED_PATH + image_id + '.tiff'
img = skimage.io.MultiImage(img_path)[1]
patches = tile2(img)
 patches3 = patches.copy()
patches4 = patches.copy()
k = 0
while k < N_TILES:
patches3[k, ] = transforms_val2(image=patches3[k, ])['image']
patches4[k, ] = transforms_val3(image=patches4[k, ])['image']
k += 1
image = np.stack([patches3, patches4])
 image = image / 255.0
pred = model.predict(image)
 isup = np.mean(pred)
 predictions72.append(isup)
 del patches, img
gc.collect()
else:
PRED_PATH = TRAIN
df_train = pd.read_csv(os.path.join(data_dir, 'train.csv'))
df = df_train.loc[:10]
df = df[['image_id','isup_grade']].copy()
predictions72 = []
for index, row in tqdm(df.iterrows() , total = df.shape[0]):
image_id = row['image_id']
img_path = PRED_PATH + image_id + '.tiff'
img = skimage.io.MultiImage(img_path)[1]
patches = tile2(img)
 patches3 = patches.copy()
patches4 = patches.copy()
k = 0
while k < N_TILES:
patches3[k, ] = transforms_val2(image=patches3[k, ])['image']
patches4[k, ] = transforms_val3(image=patches4[k, ])['image']
k += 1
image = np.stack([patches3, patches4])
 image = image / 255.0
pred = model.predict(image)
 isup = np.mean(pred)
 predictions72.append(isup)
 del patches, img
gc.collect()
del model, dummy_data, sub, pred, train, isup, image
del patches3,patches4
gc.collect()
@numba.jit
def mcc(tp, tn, fp, fn):
sup = tp * tn - fp * fn
inf =(tp + fp)*(tp + fn)*(tn + fp)*(tn + fn)
 if inf==0:
return 0
else:
return sup / np.sqrt(inf)
 def eval_mcc(y_true, y_prob, show=False):

 idx = np.argsort(y_prob)
 y_true_sort = y_true[idx]
n = y_true.shape[0]
nump = 1.0 * np.sum(y_true)
 numn = n - nump
tp = nump
tn = 0.0
fp = numn
fn = 0.0
best_mcc = 0.0
best_id = -1
prev_proba = -1
best_proba = -1
mccs = np.zeros(n)
 for i in range(n):
proba = y_prob[idx[i]]
if proba != prev_proba:
prev_proba = proba
new_mcc = mcc(tp, tn, fp, fn)
 if new_mcc >= best_mcc:
best_mcc = new_mcc
best_id = i
best_proba = proba
mccs[i] = new_mcc
if y_true_sort[i] == 1:
tp -= 1.0
fn += 1.0
else:
fp -= 1.0
tn += 1.0
if show:
y_pred =(y_prob >= best_proba ).astype(int)
 score = matthews_corrcoef(y_true, y_pred)
 plt.plot(mccs)
 return best_proba, best_mcc, y_pred
else:
return best_proba, best_mcc, None<train_model>
PREDS =(1/5)*PREDS +(1/5)*PREDS1 +(1/5)*PREDS2 +(1/5)*PREDS3 +(1/5)*PREDS4
FINAL = np.round(( 6/10)*PREDS + 
(2/60)*np.array(predictions10)+(2/60)*np.array(predictions12)+ 
(2/60)*np.array(predictions20)+(2/60)*np.array(predictions22)+ 
(2/60)*np.array(predictions30)+(2/60)*np.array(predictions32)+ 
(0.5/10)*np.array(predictions40)+(0.5/10)*np.array(predictions42)+ 
(1/60)*np.array(predictions50)+(1/60)*np.array(predictions52)+ 
(1/60)*np.array(predictions60)+(1/60)*np.array(predictions62)+ 
(1/60)*np.array(predictions70)+(1/60)*np.array(predictions72))
df['isup_grade'] = FINAL.astype(int)
 df[['image_id', 'isup_grade']].to_csv('submission.csv', index=False)
 print(df.head())
 print()
print(df.isup_grade.value_counts() )