Kaggle competition: Prostate cANcer graDe Assessment (PANDA) Challenge
<SOS> metric: QuadraticWeightedKappa Kaggle data source: prostate-cancer-grade-assessment-(panda)-challenge<categorify>
!pip install.. /input/kaggle-efficientnet-repo/efficientnet-1.0.0-py3-none-any.whl
gc.enable()
char_index = dict(( c, i)for i, c in enumerate('abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ!"
def char_encoder(str):
encoded = []
for c in str:
if c in char_index:
encoded.append(char_index[c])
 else:
encoded.append(100)
 return encoded
char_encoder('hello≈' )<feature_engineering>
sz = 256
N = 48
def tile(img):
result = []
shape = img.shape
pad0,pad1 =(sz - shape[0]%sz)%sz,(sz - shape[1]%sz)%sz
img = np.pad(img,[[pad0//2,pad0-pad0//2],[pad1//2,pad1-pad1//2],[0,0]],
constant_values=255)
 img = img.reshape(img.shape[0]//sz,sz,img.shape[1]//sz,sz,3)
 img = img.transpose(0,2,1,3,4 ).reshape(-1,sz,sz,3)
 if len(img)< N:
img = np.pad(img,[[0,N-len(img)],[0,0],[0,0],[0,0]],constant_values=255)
 idxs = np.argsort(img.reshape(img.shape[0],-1 ).sum(-1)) [:42]
img = img[idxs]
return img
def add_features(df):
df['question_text'] = df['question_text'].progress_apply(lambda x:str(x))
df['total_length'] = df['question_text'].progress_apply(len)
 df['capitals'] = df['question_text'].progress_apply(lambda comment: sum(1 for c in comment if c.isupper()))
df['caps_vs_length'] = df.progress_apply(lambda row: float(row['capitals'])/float(row['total_length']),
axis=1)
 df['num_words'] = df.question_text.str.count('\S+')
 df['num_unique_words'] = df['question_text'].progress_apply(lambda comment: len(set(w for w in comment.split())))
 df['words_vs_unique'] = df['num_unique_words'] / df['num_words']
return df
def load_and_prec() :
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 print("Train shape : ",train_df.shape)
 print("Test shape : ",test_df.shape)
 train_df["question_text"] = train_df["question_text"].progress_apply(lambda x: clean_text(x))
test_df["question_text"] = test_df["question_text"].apply(lambda x: clean_text(x))
train_df["question_text"] = train_df["question_text"].progress_apply(lambda x: replace_typical_misspell(x))
test_df["question_text"] = test_df["question_text"].apply(lambda x: replace_typical_misspell(x))
train_X_char = train_df["question_text"].progress_apply(lambda x: char_encoder(x))
test_X_char = test_df["question_text"].progress_apply(lambda x: char_encoder(x))
print("train_char shape: ", train_X_char.shape)
 print("train_char 0: ", train_X_char[0])
 train_df["question_text"] = train_df["question_text"].progress_apply(lambda x: clean_numbers(x))
test_df["question_text"] = test_df["question_text"].apply(lambda x: clean_numbers(x))
train = add_features(train_df)
 test = add_features(test_df)
 train_df["question_text"] = train_df["question_text"].apply(lambda x: x.lower())
 test_df["question_text"] = test_df["question_text"].apply(lambda x: x.lower())
 train_X = train_df["question_text"].fillna("_
test_X = test_df["question_text"].fillna("_
features = train[['caps_vs_length', 'words_vs_unique']].fillna(0)
 test_features = test[['caps_vs_length', 'words_vs_unique']].fillna(0)
 ss = StandardScaler()
ss.fit(np.vstack(( features, test_features)))
 features = ss.transform(features)
 test_features = ss.transform(test_features)
 tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 train_X = pad_sequences(train_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 train_X_char = pad_sequences(train_X_char, maxlen=char_max_len)
 test_X_char = pad_sequences(test_X_char, maxlen=char_max_len)
 train_y = train_df['target'].values
np.random.seed(SEED)
 trn_idx = np.random.permutation(len(train_X))
train_X = train_X[trn_idx]
train_y = train_y[trn_idx]
features = features[trn_idx]
train_X_char = train_X_char[trn_idx]
print("train_X word shape: ", train_X.shape)
 print("Train char shape: ", train_X_char.shape)
 print("Test char shape: ", test_X_char.shape)
 print("Train char glimpse: ", train_X_char[0])
 return train_X, test_X, train_y, train_X_char, test_X_char, features, test_features, tokenizer.word_index 
<categorify>
class ConvNet(tf.keras.Model):
def __init__(self, engine, input_shape, weights):
super(ConvNet, self ).__init__()
self.engine = engine(
 include_top=False, input_shape=input_shape, weights=weights)
 self.avg_pool2d = tf.keras.layers.GlobalAveragePooling2D()
self.dropout = tf.keras.layers.Dropout(0.5)
 self.dense_1 = tf.keras.layers.Dense(1024)
 self.dense_2 = tf.keras.layers.Dense(1)
 @tf.function
def call(self, inputs, **kwargs):
x = tf.reshape(inputs,(-1, IMG_SIZE, IMG_SIZE, 3))
x = self.engine(x)
 shape = x.shape
x = tf.reshape(x,(-1, N_TILES, shape[1], shape[2], shape[3]))
x = tf.transpose(x, perm=[0, 2, 1, 3, 4])
 x = tf.reshape(x,(-1, shape[1], N_TILES*shape[2], shape[3]))
x = self.avg_pool2d(x)
 x = self.dropout(x, training=False)
 x = self.dense_1(x)
 x = tf.nn.relu(x)
 return self.dense_2(x )
x_train, x_test, y_train, x_train_char, x_test_char, features, test_features, word_index = load_and_prec() 
<save_model>
is_ef = True
backbone_name = 'efficientnet-b0'
N_TILES = 42
IMG_SIZE = 256
if backbone_name.startswith('efficientnet'):
model_fn = getattr(efn, f'EfficientNetB{backbone_name[-1]}')
 model = ConvNet(engine=model_fn, input_shape=(IMG_SIZE, IMG_SIZE, 3), weights=None)
 dummy_data = tf.zeros(( 2 * N_TILES, IMG_SIZE, IMG_SIZE, 3), dtype=tf.float32)
 _ = model(dummy_data )
np.save("x_train",x_train)
 np.save("x_test",x_test)
 np.save("y_train",y_train)
 np.save("x_train_char", x_train_char)
 np.save("x_test_char", x_test_char)
 np.save("features",features)
 np.save("test_features",test_features)
 np.save("word_index.npy",word_index)
 np.save("char_index.npy", char_index )<load_pretrained>
model.load_weights('.. /input/tpu-training-tensorflow-iafoos-method-42x256x256x3/efficientnet-b0.h5' )
x_train = np.load("x_train.npy")
 x_test = np.load("x_test.npy")
 y_train = np.load("y_train.npy")
 x_train_char = np.load("x_train_char.npy")
 x_test_char = np.load("x_test_char.npy")
 features = np.load("features.npy")
 test_features = np.load("test_features.npy")
 word_index = np.load("word_index.npy" ).item()
char_index = np.load("char_index.npy" ).item()<concatenate>
TRAIN = '.. /input/prostate-cancer-grade-assessment/train_images/'
MASKS = '.. /input/prostate-cancer-grade-assessment/train_label_masks/'
BASE_PATH = '.. /input/prostate-cancer-grade-assessment/'
train = pd.read_csv(BASE_PATH + "train.csv")
 train.head()
x_train = np.hstack([x_train, x_train_char])
 x_test = np.hstack([x_test, x_test_char])
 del x_train_char, x_test_char
gc.collect()<feature_engineering>
sub = pd.read_csv(".. /input/prostate-cancer-grade-assessment/sample_submission.csv")
 sub.head()
seed_everything()
glove_embeddings = load_glove(word_index)
 paragram_embeddings = load_para(word_index)
 fasttext_embeddings = load_fasttext(word_index)
 embedding_matrix = np.mean([glove_embeddings, paragram_embeddings, fasttext_embeddings], axis=0)
 del glove_embeddings, paragram_embeddings, fasttext_embeddings
gc.collect()
np.shape(embedding_matrix )<split>
test = pd.read_csv(".. /input/prostate-cancer-grade-assessment/test.csv")
 test.head()
splits = list(StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=SEED ).split(x_train, y_train))
splits[:3]<choose_model_class>
TEST = '.. /input/prostate-cancer-grade-assessment/test_images/'
class CyclicLR(object):
def __init__(self, optimizer, base_lr=1e-3, max_lr=6e-3,
step_size=2000, mode='triangular', gamma=1.,
scale_fn=None, scale_mode='cycle', last_batch_iteration=-1):
if not isinstance(optimizer, Optimizer):
raise TypeError('{} is not an Optimizer'.format(
 type(optimizer ).__name__))
self.optimizer = optimizer
if isinstance(base_lr, list)or isinstance(base_lr, tuple):
if len(base_lr)!= len(optimizer.param_groups):
raise ValueError("expected {} base_lr, got {}".format(
 len(optimizer.param_groups), len(base_lr)))
 self.base_lrs = list(base_lr)
 else:
self.base_lrs = [base_lr] * len(optimizer.param_groups)
 if isinstance(max_lr, list)or isinstance(max_lr, tuple):
if len(max_lr)!= len(optimizer.param_groups):
raise ValueError("expected {} max_lr, got {}".format(
 len(optimizer.param_groups), len(max_lr)))
 self.max_lrs = list(max_lr)
 else:
self.max_lrs = [max_lr] * len(optimizer.param_groups)
 self.step_size = step_size
if mode not in ['triangular', 'triangular2', 'exp_range'] \
and scale_fn is None:
raise ValueError('mode is invalid and scale_fn is None')
 self.mode = mode
self.gamma = gamma
if scale_fn is None:
if self.mode == 'triangular':
self.scale_fn = self._triangular_scale_fn
self.scale_mode = 'cycle'
elif self.mode == 'triangular2':
self.scale_fn = self._triangular2_scale_fn
self.scale_mode = 'cycle'
elif self.mode == 'exp_range':
self.scale_fn = self._exp_range_scale_fn
self.scale_mode = 'iterations'
else:
self.scale_fn = scale_fn
self.scale_mode = scale_mode
self.batch_step(last_batch_iteration + 1)
 self.last_batch_iteration = last_batch_iteration
def batch_step(self, batch_iteration=None):
if batch_iteration is None:
batch_iteration = self.last_batch_iteration + 1
self.last_batch_iteration = batch_iteration
for param_group, lr in zip(self.optimizer.param_groups, self.get_lr()):
param_group['lr'] = lr
def _triangular_scale_fn(self, x):
return 1.
 def _triangular2_scale_fn(self, x):
return 1 /(2.**(x - 1))
def _exp_range_scale_fn(self, x):
return self.gamma**(x)
 def get_lr(self):
step_size = float(self.step_size)
 cycle = np.floor(1 + self.last_batch_iteration /(2 * step_size))
x = np.abs(self.last_batch_iteration / step_size - 2 * cycle + 1)
 lrs = []
param_lrs = zip(self.optimizer.param_groups, self.base_lrs, self.max_lrs)
 for param_group, base_lr, max_lr in param_lrs:
base_height =(max_lr - base_lr)* np.maximum(0,(1 - x))
if self.scale_mode == 'cycle':
lr = base_lr + base_height * self.scale_fn(cycle)
 else:
lr = base_lr + base_height * self.scale_fn(self.last_batch_iteration)
 lrs.append(lr)
 return lrs 
<choose_model_class>
PRED_PATH = TEST
df = sub
t_df = test
embedding_dim = 300
embedding_path = '.. /save/embedding_matrix.npy'
use_pretrained_embedding = True
hidden_size = 60
gru_len = hidden_size
Routings = 4
Num_capsule = 5
Dim_capsule = 5
dropout_p = 0.25
rate_drop_dense = 0.28
LR = 0.001
T_epsilon = 1e-7
num_classes = 30
class Embed_Layer(nn.Module):
def __init__(self, embedding_matrix=None, vocab_size=None, embedding_dim=300):
super(Embed_Layer, self ).__init__()
self.encoder = nn.Embedding(vocab_size + 1, embedding_dim)
 if use_pretrained_embedding:
self.encoder.weight.data.copy_(t.from_numpy(embedding_matrix))
def forward(self, x, dropout_p=0.25):
return nn.Dropout(p=dropout_p )(self.encoder(x))
class GRU_Layer(nn.Module):
def __init__(self):
super(GRU_Layer, self ).__init__()
self.gru = nn.GRU(input_size=300,
hidden_size=gru_len,
bidirectional=True)

def init_weights(self):
ih =(param.data for name, param in self.named_parameters() if 'weight_ih' in name)
 hh =(param.data for name, param in self.named_parameters() if 'weight_hh' in name)
 b =(param.data for name, param in self.named_parameters() if 'bias' in name)
 for k in ih:
nn.init.xavier_uniform_(k)
 for k in hh:
nn.init.orthogonal_(k)
 for k in b:
nn.init.constant_(k, 0)
 def forward(self, x):
return self.gru(x)
 class Caps_Layer(nn.Module):
def __init__(self, input_dim_capsule=gru_len * 2, num_capsule=Num_capsule, dim_capsule=Dim_capsule, \
routings=Routings, kernel_size=(9, 1), share_weights=True,
activation='default', **kwargs):
super(Caps_Layer, self ).__init__(**kwargs)
 self.num_capsule = num_capsule
self.dim_capsule = dim_capsule
self.routings = routings
self.kernel_size = kernel_size
self.share_weights = share_weights
if activation == 'default':
self.activation = self.squash
else:
self.activation = nn.ReLU(inplace=True)
 if self.share_weights:
self.W = nn.Parameter(
 nn.init.xavier_normal_(t.empty(1, input_dim_capsule, self.num_capsule * self.dim_capsule)))
 else:
self.W = nn.Parameter(
 t.randn(BATCH_SIZE, input_dim_capsule, self.num_capsule * self.dim_capsule))
def forward(self, x):
if self.share_weights:
u_hat_vecs = t.matmul(x, self.W)
 else:
print('add later')
 batch_size = x.size(0)
 input_num_capsule = x.size(1)
 u_hat_vecs = u_hat_vecs.view(( batch_size, input_num_capsule,
self.num_capsule, self.dim_capsule))
u_hat_vecs = u_hat_vecs.permute(0, 2, 1, 3)
 b = t.zeros_like(u_hat_vecs[:, :, :, 0])
 for i in range(self.routings):
b = b.permute(0, 2, 1)
 c = F.softmax(b, dim=2)
 c = c.permute(0, 2, 1)
 b = b.permute(0, 2, 1)
 outputs = self.activation(t.einsum('bij,bijk->bik',(c, u_hat_vecs)))
 if i < self.routings - 1:
b = t.einsum('bik,bijk->bij',(outputs, u_hat_vecs))
return outputs
def squash(self, x, axis=-1):
s_squared_norm =(x ** 2 ).sum(axis, keepdim=True)
 scale = t.sqrt(s_squared_norm + T_epsilon)
 return x / scale 
<categorify>
transforms = albumentations.Compose([
albumentations.Transpose(p=0.5),
albumentations.VerticalFlip(p=0.5),
albumentations.HorizontalFlip(p=0.5),
] )
print(np.shape(np_utils.to_categorical(x_train[:512, maxlen:])))
 print(np.shape(x_train[:512, maxlen:]))
print(np.shape(x_train[:512, 5])) 
<categorify>
if os.path.exists(PRED_PATH):
predictions = []
for index, row in tqdm(df.iterrows() , total = df.shape[0]):
image_id = row['image_id']
img_path = PRED_PATH + image_id + '.tiff'
img = skimage.io.MultiImage(img_path)[1]
patches = tile(img)
 patches1 = patches.copy()
patches2 = patches.copy()
k = 0
while k < 42:
patches1[k, ] = transforms(image=patches1[k, ])['image']
patches2[k, ] = transforms(image=patches2[k, ])['image']
k += 1
image = np.stack([patches, patches1, patches2])
 image = image / 255.0
pred = model.predict(image)
 isup = np.round(np.mean(pred))
if isup < 0:
isup = 0
if isup > 5:
isup = 5
predictions.append(int(isup))
del patches, img
gc.collect()
<categorify><EOS> 
if os.path.exists(PRED_PATH):
sub['isup_grade'] = predictions
sub.to_csv("submission.csv", index=False)
 else:
sub.to_csv("submission.csv", index=False )