Kaggle competition: Prostate cANcer graDe Assessment (PANDA) Challenge
<SOS> metric: QuadraticWeightedKappa Kaggle data source: prostate-cancer-grade-assessment-(panda)-challenge<split>
!pip install.. /input/kaggle-efficientnet-repo/efficientnet-1.0.0-py3-none-any.whl
gc.enable()
df_ds_train, df_ds_test = train_test_split_by_match(df_ds, test_size=test_size )<prepare_x_and_y>
sz = 256
N = 48
def tile(img):
result = []
shape = img.shape
pad0,pad1 =(sz - shape[0]%sz)%sz,(sz - shape[1]%sz)%sz
img = np.pad(img,[[pad0//2,pad0-pad0//2],[pad1//2,pad1-pad1//2],[0,0]],
constant_values=255)
 img = img.reshape(img.shape[0]//sz,sz,img.shape[1]//sz,sz,3)
 img = img.transpose(0,2,1,3,4 ).reshape(-1,sz,sz,3)
 if len(img)< N:
img = np.pad(img,[[0,N-len(img)],[0,0],[0,0],[0,0]],constant_values=255)
 idxs = np.argsort(img.reshape(img.shape[0],-1 ).sum(-1)) [:42]
img = img[idxs]
return img
print('Processing train Duo-Squad dataset...')
 with timeit() :
X_ds_train, y_ds_train = process_ds(df_ds_train)
 print('
Processing test Duo-Squad dataset...')
 with timeit() :
X_ds_test, y_ds_test = process_ds(df_ds_test )<train_model>
class ConvNet(tf.keras.Model):
def __init__(self, engine, input_shape, weights):
super(ConvNet, self ).__init__()
self.engine = engine(
 include_top=False, input_shape=input_shape, weights=weights)
 self.avg_pool2d = tf.keras.layers.GlobalAveragePooling2D()
self.dropout = tf.keras.layers.Dropout(0.5)
 self.dense_1 = tf.keras.layers.Dense(1024)
 self.dense_2 = tf.keras.layers.Dense(1)
 @tf.function
def call(self, inputs, **kwargs):
x = tf.reshape(inputs,(-1, IMG_SIZE, IMG_SIZE, 3))
x = self.engine(x)
 shape = x.shape
x = tf.reshape(x,(-1, N_TILES, shape[1], shape[2], shape[3]))
x = tf.transpose(x, perm=[0, 2, 1, 3, 4])
 x = tf.reshape(x,(-1, shape[1], N_TILES*shape[2], shape[3]))
x = self.avg_pool2d(x)
 x = self.dropout(x, training=False)
 x = self.dense_1(x)
 x = tf.nn.relu(x)
 return self.dense_2(x )
scaler_ds = StandardScaler()
print('Fitting Duo-Squad standard scaling...')
 with timeit() :
scaler_ds.fit(X_ds_train )<choose_model_class>
is_ef = True
backbone_name = 'efficientnet-b0'
N_TILES = 42
IMG_SIZE = 256
if backbone_name.startswith('efficientnet'):
model_fn = getattr(efn, f'EfficientNetB{backbone_name[-1]}')
 model = ConvNet(engine=model_fn, input_shape=(IMG_SIZE, IMG_SIZE, 3), weights=None)
 dummy_data = tf.zeros(( 2 * N_TILES, IMG_SIZE, IMG_SIZE, 3), dtype=tf.float32)
 _ = model(dummy_data )
def get_model(input_shape):
model = keras.models.Sequential()
model.add(L.InputLayer(input_shape=input_shape))
model.add(L.Dense(64, activation='relu'))
model.add(L.BatchNormalization())
 model.add(L.Dense(32, activation='relu'))
model.add(L.BatchNormalization())
 model.add(L.Dense(16, activation='relu'))
model.add(L.Dense(1, activation='sigmoid', kernel_initializer='normal'))
model.compile(loss='mean_absolute_error', optimizer='adam')
 return model<choose_model_class>
model.load_weights('.. /input/tpu-training-tensorflow-iafoos-method-42x256x256x3/efficientnet-b0.h5' )
model_ds = get_model([len(X_ds_train.columns)] )<train_model>
TRAIN = '.. /input/prostate-cancer-grade-assessment/train_images/'
MASKS = '.. /input/prostate-cancer-grade-assessment/train_label_masks/'
BASE_PATH = '.. /input/prostate-cancer-grade-assessment/'
train = pd.read_csv(BASE_PATH + "train.csv")
 train.head()
validation_data = []
if env != 'kaggle':
validation_data = [
scaler_ds.transform(X_ds_test),
y_ds_test
]
fit(
 model_ds,
scaler_ds.transform(X_ds_train),
y_ds_train,
epochs=100,
validation_data=validation_data,
batch_size=batch_size,
callbacks=[
EarlyStopping('loss', min_delta=0.0001, patience=20)
 ]
)<compute_train_metric>
sub = pd.read_csv(".. /input/prostate-cancer-grade-assessment/sample_submission.csv")
 sub.head()
if env != 'kaggle':
y_ds_pred = model_ds.predict(scaler_ds.transform(X_ds_test))
y_ds_pred_post = postprocess(df_ds_test, y_ds_pred)
 print('MAE DS:', mean_absolute_error(y_ds_test, y_ds_pred_post))<split>
test = pd.read_csv(".. /input/prostate-cancer-grade-assessment/test.csv")
 test.head()
df_solo_train, df_solo_test = train_test_split_by_match(df_solo, test_size=test_size )<prepare_x_and_y>
TEST = '.. /input/prostate-cancer-grade-assessment/test_images/'
print('Processing train Solo dataset...')
 with timeit() :
X_solo_train, y_solo_train = process_solo(df_solo_train)
 print('
Processing train Solo dataset...')
 with timeit() :
X_solo_test, y_solo_test = process_solo(df_solo_test )<normalization>
PRED_PATH = TEST
df = sub
t_df = test
scaler_solo = StandardScaler()
scaler_solo.fit(X_solo_train )<choose_model_class>
transforms = albumentations.Compose([
albumentations.Transpose(p=0.5),
albumentations.VerticalFlip(p=0.5),
albumentations.HorizontalFlip(p=0.5),
] )
model_solo = get_model([len(X_solo_train.columns)] )<train_model>
if os.path.exists(PRED_PATH):
predictions = []
for index, row in tqdm(df.iterrows() , total = df.shape[0]):
image_id = row['image_id']
img_path = PRED_PATH + image_id + '.tiff'
img = skimage.io.MultiImage(img_path)[1]
patches = tile(img)
 patches1 = patches.copy()
patches2 = patches.copy()
k = 0
while k < 42:
patches1[k, ] = transforms(image=patches1[k, ])['image']
patches2[k, ] = transforms(image=patches2[k, ])['image']
k += 1
image = np.stack([patches, patches1, patches2])
 image = image / 255.0
pred = model.predict(image)
 isup = np.round(np.mean(pred))
if isup < 0:
isup = 0
if isup > 5:
isup = 5
predictions.append(int(isup))
del patches, img
gc.collect()
<compute_test_metric><EOS> 
if os.path.exists(PRED_PATH):
sub['isup_grade'] = predictions
sub.to_csv("submission.csv", index=False)
 else:
sub.to_csv("submission.csv", index=False )