Kaggle competition: Prostate cANcer graDe Assessment (PANDA) Challenge
<SOS> metric: QuadraticWeightedKappa Kaggle data source: prostate-cancer-grade-assessment-(panda)-challenge<categorify>
!pip install.. /input/installation/Keras_Applications-1.0.8-py3-none-any.whl -f./ --no-index --no-deps
!pip install.. /input/install-2/efficientnet-1.1.0-py3-none-any.whl -f./ --no-index --no-deps
def transform_ts(ts, n_dim=160, min_max=(-1,1)) :
ts_std = min_max_transf(ts, min_data=min_num, max_data=max_num)
 bucket_size = int(sample_size / n_dim)
 new_ts = []
for i in range(0, sample_size, bucket_size):
ts_range = ts_std[i:i + bucket_size]
mean = ts_range.mean()
std = ts_range.std()
std_top = mean + std
std_bot = mean - std
percentil_calc = np.percentile(ts_range, [0, 1, 25, 50, 75, 99, 100])
 max_range = percentil_calc[-1] - percentil_calc[0]
relative_percentile = percentil_calc - mean
new_ts.append(np.concatenate([np.asarray([mean, std, std_top, std_bot, max_range]),percentil_calc, relative_percentile]))
return np.asarray(new_ts )<prepare_x_and_y>
warnings.filterwarnings('ignore')
 print('tensorflow version:', tf.__version__ )
def prep_data(start, end):
praq_train = pq.read_pandas('.. /input/train.parquet', columns=[str(i)for i in range(start, end)] ).to_pandas()
X = []
y = []
for id_measurement in tqdm(df_train.index.levels[0].unique() [int(start/3):int(end/3)]):
X_signal = []
for phase in [0,1,2]:
signal_id, target = df_train.loc[id_measurement].loc[phase]
if phase == 0:
y.append(target)
 X_signal.append(transform_ts(praq_train[str(signal_id)]))
X_signal = np.concatenate(X_signal, axis=1)
 X.append(X_signal)
 X = np.asarray(X)
 y = np.asarray(y)
 return X, y<prepare_x_and_y>
gpus = tf.config.experimental.list_physical_devices('GPU')
 if gpus:
try:
for gpu in gpus:
tf.config.experimental.set_memory_growth(gpu, True)
 logical_gpus = tf.config.experimental.list_logical_devices('GPU')
 print(len(gpus), "Physical GPUs,", len(logical_gpus), "Logical GPUs")
 except RuntimeError as e:
print(e)
 else:
print('no gpus' )
X = np.load(".. /input/5-fold-lstm-attention-fully-commented-0-694/X.npy")
 y = np.load(".. /input/5-fold-lstm-attention-fully-commented-0-694/y.npy" )<import_modules>
DATA_PATH = '.. /input/prostate-cancer-grade-assessment'
MODELS_PATH = '.'
IMG_SIZE = 128
SEQ_LEN = 36
TARGET_SIZE = [528, 528]
BATCH_SIZE = 16
MDL_VERSION = 'v0'
SEED = 80
LEVEL = 1
try:
except: pass
class LayerNormalization(Layer):
def __init__(self, eps=1e-6, **kwargs):
self.eps = eps
super(LayerNormalization, self ).__init__(**kwargs)
 def build(self, input_shape):
self.gamma = self.add_weight(name='gamma', shape=input_shape[-1:],
initializer=Ones() , trainable=True)
 self.beta = self.add_weight(name='beta', shape=input_shape[-1:],
initializer=Zeros() , trainable=True)
 super(LayerNormalization, self ).build(input_shape)
 def call(self, x):
mean = K.mean(x, axis=-1, keepdims=True)
 std = K.std(x, axis=-1, keepdims=True)
 return self.gamma *(x - mean)/(std + self.eps)+ self.beta
def compute_output_shape(self, input_shape):
return input_shape
class ScaledDotProductAttention() :
def __init__(self, d_model, attn_dropout=0.1):
self.temper = np.sqrt(d_model)
 self.dropout = Dropout(attn_dropout)
 def __call__(self, q, k, v, mask):
attn = Lambda(lambda x:K.batch_dot(x[0],x[1],axes=[2,2])/self.temper )([q, k])
 if mask is not None:
mmask = Lambda(lambda x:(-1e+10)*(1-x))(mask)
 attn = Add()([attn, mmask])
 attn = Activation('softmax' )(attn)
 attn = self.dropout(attn)
 output = Lambda(lambda x:K.batch_dot(x[0], x[1]))([attn, v])
 return output, attn
class MultiHeadAttention() :
def __init__(self, n_head, d_model, d_k, d_v, dropout, mode=0, use_norm=True):
self.mode = mode
self.n_head = n_head
self.d_k = d_k
self.d_v = d_v
self.dropout = dropout
if mode == 0:
self.qs_layer = Dense(n_head*d_k, use_bias=False)
 self.ks_layer = Dense(n_head*d_k, use_bias=False)
 self.vs_layer = Dense(n_head*d_v, use_bias=False)
 elif mode == 1:
self.qs_layers = []
self.ks_layers = []
self.vs_layers = []
for _ in range(n_head):
self.qs_layers.append(TimeDistributed(Dense(d_k, use_bias=False)))
 self.ks_layers.append(TimeDistributed(Dense(d_k, use_bias=False)))
 self.vs_layers.append(TimeDistributed(Dense(d_v, use_bias=False)))
 self.attention = ScaledDotProductAttention(d_model)
 self.layer_norm = LayerNormalization() if use_norm else None
self.w_o = TimeDistributed(Dense(d_model))
def __call__(self, q, k, v, mask=None):
d_k, d_v = self.d_k, self.d_v
n_head = self.n_head
if self.mode == 0:
qs = self.qs_layer(q)
 ks = self.ks_layer(k)
 vs = self.vs_layer(v)
 def reshape1(x):
s = tf.shape(x)
 x = tf.reshape(x, [s[0], s[1], n_head, d_k])
 x = tf.transpose(x, [2, 0, 1, 3])
 x = tf.reshape(x, [-1, s[1], d_k])
 return x
qs = Lambda(reshape1 )(qs)
 ks = Lambda(reshape1 )(ks)
 vs = Lambda(reshape1 )(vs)
 if mask is not None:
mask = Lambda(lambda x:K.repeat_elements(x, n_head, 0))(mask)
 head, attn = self.attention(qs, ks, vs, mask=mask)
 def reshape2(x):
s = tf.shape(x)
 x = tf.reshape(x, [n_head, -1, s[1], s[2]])
 x = tf.transpose(x, [1, 2, 0, 3])
 x = tf.reshape(x, [-1, s[1], n_head*d_v])
 return x
head = Lambda(reshape2 )(head)
 elif self.mode == 1:
heads = []; attns = []
for i in range(n_head):
qs = self.qs_layers[i](q)
 ks = self.ks_layers[i](k)
 vs = self.vs_layers[i](v)
 head, attn = self.attention(qs, ks, vs, mask)
 heads.append(head); attns.append(attn)
 head = Concatenate()(heads)if n_head > 1 else heads[0]
attn = Concatenate()(attns)if n_head > 1 else attns[0]
outputs = self.w_o(head)
 outputs = Dropout(self.dropout )(outputs)
 if not self.layer_norm: return outputs, attn
outputs = Add()([outputs, q])
 return self.layer_norm(outputs), attn
class PositionwiseFeedForward() :
def __init__(self, d_hid, d_inner_hid, dropout=0.1):
self.w_1 = Conv1D(d_inner_hid, 1, activation='relu')
 self.w_2 = Conv1D(d_hid, 1)
 self.layer_norm = LayerNormalization()
self.dropout = Dropout(dropout)
 def __call__(self, x):
output = self.w_1(x)
 output = self.w_2(output)
 output = self.dropout(output)
 output = Add()([output, x])
 return self.layer_norm(output)
 class EncoderLayer() :
def __init__(self, d_model, d_inner_hid, n_head, d_k, d_v, dropout=0.1):
self.self_att_layer = MultiHeadAttention(n_head, d_model, d_k, d_v, dropout=dropout)
 self.pos_ffn_layer = PositionwiseFeedForward(d_model, d_inner_hid, dropout=dropout)
 def __call__(self, enc_input, mask=None):
output, slf_attn = self.self_att_layer(enc_input, enc_input, enc_input, mask=mask)
 output = self.pos_ffn_layer(output)
 return output, slf_attn
def GetPosEncodingMatrix(max_len, d_emb):
pos_enc = np.array([
[pos / np.power(10000, 2 *(j // 2)/ d_emb)for j in range(d_emb)]
if pos != 0 else np.zeros(d_emb)
 for pos in range(max_len)
 ])
 pos_enc[1:, 0::2] = np.sin(pos_enc[1:, 0::2])
 pos_enc[1:, 1::2] = np.cos(pos_enc[1:, 1::2])
 return pos_enc
def GetPadMask(q, k):
ones = K.expand_dims(K.ones_like(q, 'float32'), -1)
 mask = K.cast(K.expand_dims(K.not_equal(k, 0), 1), 'float32')
 mask = K.batch_dot(ones, mask, axes=[2,1])
 return mask
def GetSubMask(s):
len_s = tf.shape(s)[1]
bs = tf.shape(s)[:1]
mask = K.cumsum(tf.eye(len_s, batch_shape=bs), 1)
 return mask
class Encoder() :
def __init__(self, d_model, d_inner_hid, n_head, d_k, d_v, \
layers=6, dropout=0.1, pos_emb=None):
self.pos_layer = pos_emb
self.dropout = Dropout(dropout)
 self.layers = [EncoderLayer(d_model, d_inner_hid, n_head, d_k, d_v, dropout)for _ in range(layers)]
def __call__(self, src_seq, src_pos, return_att=False, active_layers=999):
x = src_seq
if src_pos is not None:
pos = self.pos_layer(src_pos)
 x = Add()([x, pos])
 x = self.dropout(x)
 if return_att: atts = []
mask = None
for enc_layer in self.layers[:active_layers]:
x, att = enc_layer(x, mask)
 if return_att: atts.append(att)
 return(x, atts)if return_att else x
class Transformer() :
def __init__(self, len_limit, d_model=X.shape[-1], \
d_inner_hid=512, n_head=8, d_k=64, d_v=64, layers=2, dropout=0.1, \
share_word_emb=False, **kwargs):
self.name = 'Transformer'
self.len_limit = len_limit
self.src_loc_info = True
self.d_model = d_model
d_emb = d_model
pos_emb = Embedding(len_limit, d_emb, trainable=False, \
weights=[GetPosEncodingMatrix(len_limit, d_emb)])
 self.encoder = Encoder(d_model, d_inner_hid, n_head, d_k, d_v, layers, dropout, \
pos_emb=pos_emb)
 def get_pos_seq(self, x):
pos = K.cumsum(K.ones_like(x[...,0], 'int32'), 1)
 return pos
def compile(self, active_layers=999):
src_seq = Input(shape=(X.shape[1],X.shape[2]))
src_pos = Lambda(self.get_pos_seq )(src_seq)
 if not self.src_loc_info: src_pos = None
x = self.encoder(src_seq, src_pos, active_layers=active_layers)
 x = GlobalAveragePooling1D()(x)
 outp = Dense(1, activation="sigmoid" )(x)
 self.model = Model(inputs=src_seq, outputs=outp)
 self.model.compile(loss='binary_crossentropy', optimizer='adam', metrics=[matthews_correlation] )<split>

 def get_axis_max_min(array, axis=0):
one_axis = list(( array != 255 ).sum(axis=tuple([x for x in(0, 1, 2)if x != axis])))
 axis_min = next(( i for i, x in enumerate(one_axis)if x), 0)
 axis_max = len(one_axis)- next(( i for i, x in enumerate(one_axis[::-1])if x), 0)
 return axis_min, axis_max
class DataGenPanda(Sequence):
def __init__(self, imgs_path, df, batch_size=32,
mode='fit', shuffle=False, aug=None,
seq_len=SEQ_LEN, img_size=IMG_SIZE, n_classes=6):
self.imgs_path = imgs_path
self.df = df
self.shuffle = shuffle
self.mode = mode
self.aug = aug
self.batch_size = batch_size
self.img_size = img_size
self.seq_len = seq_len
self.n_classes = n_classes
self.side = int(seq_len **.5)
 self.on_epoch_end()
def __len__(self):
return int(np.floor(len(self.df)/ self.batch_size))
def on_epoch_end(self):
self.indexes = np.arange(len(self.df))
if self.shuffle:
np.random.shuffle(self.indexes)
 def __getitem__(self, index):
X = np.zeros(( self.batch_size, self.side * self.img_size, self.side * self.img_size, 3), dtype=np.float32)
 imgs_batch = self.df[index * self.batch_size :(index + 1)* self.batch_size]['image_id'].values
for i, img_name in enumerate(imgs_batch):
img_path = '{}/{}.tiff'.format(self.imgs_path, img_name)
 img_patches = self.get_patches(img_path)
 X[i, ] = self.glue_to_one(img_patches)
 X = tf.image.convert_image_dtype(X, tf.float32)
 X = tf.image.resize(X, size = TARGET_SIZE)
 if self.mode == 'fit':
y = np.zeros(( self.batch_size, self.n_classes), dtype=np.float32)
 lbls_batch = self.df[index * self.batch_size :(index + 1)* self.batch_size]['isup_grade'].values
for i in range(self.batch_size):
y[i, lbls_batch[i]] = 1
return X, y
elif self.mode == 'predict':
return X
else:
raise AttributeError('mode parameter error')
 def get_patches(self, img_path):
num_patches = self.seq_len
p_size = self.img_size
img = skimage.io.MultiImage(img_path)[LEVEL]
a0min, a0max = get_axis_max_min(img, axis=0)
 a1min, a1max = get_axis_max_min(img, axis=1)
 img = img[a0min:a0max, a1min:a1max, :].astype(np.float32)/ 255
if self.aug:
img = self.aug(image=img)['image']
pad0, pad1 =(p_size - img.shape[0] % p_size)% p_size,(p_size - img.shape[1] % p_size)% p_size
img = np.pad(
 img,
[
[pad0 // 2, pad0 - pad0 // 2],
[pad1 // 2, pad1 - pad1 // 2],
[0, 0]
],
constant_values=1 
)
 img = img.reshape(img.shape[0] // p_size, p_size, img.shape[1] // p_size, p_size, 3)
 img = img.transpose(0, 2, 1, 3, 4 ).reshape(-1, p_size, p_size, 3)
 if len(img)< num_patches:
img = np.pad(
 img,
[
[0, num_patches - len(img)],
[0, 0],
[0, 0],
[0, 0]
],
constant_values=1 
)
 idxs = np.argsort(img.reshape(img.shape[0], -1 ).sum(-1)) [:num_patches]
return np.array(img[idxs])
 def glue_to_one(self, imgs_seq):
img_glue = np.zeros(( self.img_size * self.side, self.img_size * self.side, 3), dtype=np.float32)
 for i, ptch in enumerate(imgs_seq):
x = i // self.side
y = i % self.side
img_glue[x * self.img_size :(x + 1)* self.img_size,
y * self.img_size :(y + 1)* self.img_size, :] = ptch
return img_glue
splits = list(StratifiedKFold(n_splits=N_SPLITS, shuffle=True, random_state=2019 ).split(X, y))
preds_val = []
y_val = []
for idx,(train_idx, val_idx)in enumerate(splits):
K.clear_session()
print("Beginning fold {}".format(idx+1))
train_X, train_y, val_X, val_y = X[train_idx], y[train_idx], X[val_idx], y[val_idx]
model = Transformer(train_X.shape[1], layers=1,d_inner_hid=256)
 model.compile()
model = model.model
ckpt = ModelCheckpoint('weights_{}.h5'.format(idx), save_best_only=True, save_weights_only=True, verbose=1, monitor='val_matthews_correlation', mode='max')
 model.fit(train_X, train_y, batch_size=128, epochs=100, validation_data=[val_X, val_y], callbacks=[ckpt])
 model.load_weights('weights_{}.h5'.format(idx))
preds_val.append(model.predict(val_X, batch_size=512))
y_val.append(val_y)
 preds_val = np.concatenate(preds_val)[...,0]
y_val = np.concatenate(y_val)
 preds_val.shape, y_val.shape<compute_train_metric>
train = pd.read_csv('{}/train.csv'.format(DATA_PATH))
print('train: ', train.shape, '| unique ids:', sum(train['isup_grade'].value_counts()))
X_train, X_val = train_test_split(train, test_size=.2, stratify=train['isup_grade'], random_state=SEED)
 lbl_value_counts = X_train['isup_grade'].value_counts()
class_weights = {i: max(lbl_value_counts)/ v for i, v in lbl_value_counts.items() }
print('classes weigths:', class_weights )
def threshold_search(y_true, y_proba):
best_threshold = 0
best_score = 0
for threshold in tqdm([i * 0.01 for i in range(100)]):
score = matthews_correlation(y_true,(y_proba > threshold ).astype(np.float64),K=np)
 if score > best_score:
best_threshold = threshold
best_score = score
search_result = {'threshold': best_threshold, 'matthews_correlation': best_score}
print(search_result)
 return search_result<compute_test_metric>
aug = albu.Compose(
 [
albu.OneOf([albu.RandomBrightness(limit=.15), albu.RandomContrast(limit=.3), albu.RandomGamma() ], p=.25),
albu.HorizontalFlip(p=.25),
albu.VerticalFlip(p=.25),
albu.ShiftScaleRotate(shift_limit=.1, scale_limit=.1, rotate_limit=20, p=.25)
 ] 
)
 train_datagen = DataGenPanda(
 imgs_path='{}/train_images'.format(DATA_PATH),
df=X_train,
batch_size=BATCH_SIZE,
mode='fit',
shuffle=True,
aug=aug,
seq_len=SEQ_LEN,
img_size=IMG_SIZE,
n_classes=6 
)
 val_datagen = DataGenPanda(
 imgs_path='{}/train_images'.format(DATA_PATH),
df=X_val,
batch_size=BATCH_SIZE,
mode='fit',
shuffle=False,
aug=None,
seq_len=SEQ_LEN,
img_size=IMG_SIZE,
n_classes=6
)
best_threshold = threshold_search(y_val, preds_val)['threshold']<load_from_csv>
MODEL_DIR = '.. /input/pandamodels/'
models = {'B0': 'EfficientNetB0.h5',
'B2':'EfficientNetB2.h5',
'B4':'EfficientNetB4.h5',
'B6':'EfficientNetB6.h5',
'Xception':'Xception.h5'}
model = tf.keras.models.load_model(MODEL_DIR + models['B6'], compile=False)
 print('Model loaded:', model )
%%time
meta_test = pd.read_csv('.. /input/vsb-power-line-fault-detection/metadata_test.csv' )<drop_column>
test = pd.read_csv('{}/test.csv'.format(DATA_PATH))
preds = [[0] * 6] * len(test)
 if os.path.exists('.. /input/prostate-cancer-grade-assessment/test_images'):
print('Inference on test images...')
 subm_datagen = DataGenPanda(
 imgs_path='{}/test_images'.format(DATA_PATH),
df=test,
batch_size=1,
mode='predict',
shuffle=False,
aug=None,
seq_len=SEQ_LEN,
img_size=IMG_SIZE,
n_classes=6 
)
 preds = model.predict_generator(subm_datagen)
 print('Total preds test:', len(preds))
else:
print('Inference on validation images...')
 preds = model.predict_generator(val_datagen.__getitem__(1)) [:len(preds)]
<load_pretrained><EOS> 
test['isup_grade'] = np.argmax(preds, axis=1)
 test.drop('data_provider',inplace= True, axis=1)
 test.to_csv('submission.csv', index=False)
 test.head()