Kaggle competition: Plant Pathology 2020 - FGVC7
<SOS> metric: MCAUC Kaggle data source: plant-pathology-2020-fgvc7<train_model>
!pip install -q efficientnet
def train_and_infer(rnn, STRATIFY=True, FOLDS=4, EPOCHS=50, BATCH_SIZE=64,
REPEATS=3, SEED=34, VERBOSE=2):
public_df = test.query("seq_length == 107" ).copy()
private_df = test.query("seq_length == 130" ).copy()
private_preds = np.zeros(( private_df.shape[0], 130, 5))
public_preds = np.zeros(( public_df.shape[0], 107, 5))
public_inputs = preprocess_inputs(public_df)
 private_inputs = preprocess_inputs(private_df)
 holdouts = []
holdout_preds = []
histories = []
gkf = GroupKFold(n_splits=FOLDS)
 kf=KFold(n_splits=FOLDS, random_state=SEED)
 kmeans_model = KMeans(n_clusters=200, random_state=SEED ).fit(preprocess_inputs(train)[:,:,0])
 train['cluster_id'] = kmeans_model.labels_
for _ in range(REPEATS):
for f,(train_index, val_index)in enumerate(( gkf if STRATIFY else kf ).split(train,
train['reactivity'], train['cluster_id'] if STRATIFY else None)) :
lr_callback = tf.keras.callbacks.ReduceLROnPlateau(patience=8,
factor=.1,
verbose=VERBOSE)
 save = tf.keras.callbacks.ModelCheckpoint(f'model-{f}.h5')
 epsilon =.1
sample_weighting = np.log1p(train.iloc[train_index]['signal_to_noise'] + epsilon)/ 2
trn = train.iloc[train_index]
trn_ = preprocess_inputs(trn)
 trn_labs = np.array(trn[target_cols].values.tolist() ).transpose(( 0, 2, 1))
val = train.iloc[val_index]
val_all = preprocess_inputs(val)
 val = val[val.SN_filter == 1]
val_ = preprocess_inputs(val)
 val_labs = np.array(val[target_cols].values.tolist() ).transpose(( 0, 2, 1))
model = build_model(rnn=rnn)
 model_short = build_model(rnn=rnn,seq_len=107, pred_len=107)
 model_long = build_model(rnn=rnn,seq_len=130, pred_len=130)
 history = model.fit(
 trn_, trn_labs,
validation_data =(val_, val_labs),
batch_size=BATCH_SIZE,
epochs=EPOCHS,
sample_weight=sample_weighting,
callbacks=[save, lr_callback],
verbose=VERBOSE 
)
 histories.append(history)
 model.load_weights(f'model-{f}.h5')
 model_short.load_weights(f'model-{f}.h5')
 model_long.load_weights(f'model-{f}.h5')
 holdouts.append(train.iloc[val_index])
 holdout_preds.append(model.predict(val_all))
public_preds += model_short.predict(public_inputs)/(FOLDS * REPEATS)
 private_preds += model_long.predict(private_inputs)/(FOLDS * REPEATS)
 del model, model_short, model_long
return holdouts, holdout_preds, public_df, public_preds, private_df, private_preds, histories<split>
import math, re, os
import numpy as np
import pandas as pd
from matplotlib import pyplot as plt
from kaggle_datasets import KaggleDatasets
import tensorflow as tf
import tensorflow.keras.layers as L
import efficientnet.tfkeras as efn
from sklearn import metrics
from sklearn.model_selection import train_test_split
gru_holdouts, gru_holdout_preds, public_df, gru_public_preds, private_df, gru_private_preds, gru_histories = train_and_infer(rnn='gru' )<split>
AUTO = tf.data.experimental.AUTOTUNE
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 GCS_DS_PATH = KaggleDatasets().get_gcs_path()
EPOCHS = 80
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
lstm_holdouts, lstm_holdout_preds, public_df, lstm_public_preds, private_df, lstm_private_preds, lstm_histories = train_and_infer(rnn='lstm' )<prepare_output>
def format_path(st):
return GCS_DS_PATH + '/images/' + st + '.jpg'
def format_predictions(test_df, test_preds, val=False):
preds = []
for df, preds_ in zip(test_df, test_preds):
for i, uid in enumerate(df['id']):
single_pred = preds_[i]
single_df = pd.DataFrame(single_pred, columns=target_cols)
 single_df['id_seqpos'] = [f'{uid}_{x}' for x in range(single_df.shape[0])]
if val: single_df['SN_filter'] = df[df['id'] == uid].SN_filter.values[0]
preds.append(single_df)
 return pd.concat(preds ).groupby('id_seqpos' ).mean().reset_index() if AUGMENT else pd.concat(preds )<load_from_disk>
train = pd.read_csv('/kaggle/input/plant-pathology-2020-fgvc7/train.csv')
 test = pd.read_csv('/kaggle/input/plant-pathology-2020-fgvc7/test.csv')
 sub = pd.read_csv('/kaggle/input/plant-pathology-2020-fgvc7/sample_submission.csv')
 train_paths = train.image_id.apply(format_path ).values
test_paths = test.image_id.apply(format_path ).values
train_labels = train.loc[:, 'healthy':].values
def get_error(preds):
val = pd.read_json('.. /input/stanford-covid-vaccine/train.json', lines=True)
 val_data = []
for mol_id in val['id'].unique() :
sample_data = val.loc[val['id'] == mol_id]
sample_seq_length = sample_data.seq_length.values[0]
for i in range(68):
sample_dict = {
'id_seqpos' : sample_data['id'].values[0] + '_' + str(i),
'reactivity_gt' : sample_data['reactivity'].values[0][i],
'deg_Mg_pH10_gt' : sample_data['deg_Mg_pH10'].values[0][i],
'deg_Mg_50C_gt' : sample_data['deg_Mg_50C'].values[0][i],
}
val_data.append(sample_dict)
 val_data = pd.DataFrame(val_data)
 val_data = val_data.merge(preds, on='id_seqpos')
 rmses = []
mses = []
for col in ['reactivity', 'deg_Mg_pH10', 'deg_Mg_50C']:
rmse =(( val_data[col] - val_data[col+'_gt'])** 2 ).mean() **.5
mse =(( val_data[col] - val_data[col+'_gt'])** 2 ).mean()
rmses.append(rmse)
 mses.append(mse)
 print(col, rmse, mse)
 print(np.mean(rmses), np.mean(mses))
print('' )<predict_on_test>
def decode_image(filename, label=None, image_size=(512, 512)) :
bits = tf.io.read_file(filename)
 image = tf.image.decode_jpeg(bits, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.image.rgb_to_yuv(image)
 image = tf.image.resize(image, image_size)
 if label is None:
return image
else:
return image, label
def data_augment(image, label=None):
image = tf.image.random_flip_left_right(image)
 image = tf.image.random_flip_up_down(image)
 if label is None:
return image
else:
return image, label
gru_preds = [gru_public_preds, gru_private_preds]
lstm_preds = [gru_public_preds, gru_private_preds]
test_df = [public_df, private_df]
gru_preds = format_predictions(test_df, gru_preds)
 lstm_preds = format_predictions(test_df, lstm_preds )<init_hyperparams>
test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(test_paths)
.map(decode_image, num_parallel_calls=AUTO)
.map(data_augment, num_parallel_calls=AUTO)
.batch(BATCH_SIZE)
 )
gru_weight =.5
lstm_weight =.5<prepare_output>
def build_lrfn(lr_start=0.00001, lr_max=0.00005,
lr_min=0.000001, lr_rampup_epochs=3,
lr_sustain_epochs=2, lr_exp_decay=.86):
lr_max = lr_max * strategy.num_replicas_in_sync
def lrfn(epoch):
if epoch < lr_rampup_epochs:
lr =(lr_max - lr_start)/ lr_rampup_epochs * epoch + lr_start
elif epoch < lr_rampup_epochs + lr_sustain_epochs:
lr = lr_max
else:
lr =(lr_max - lr_min)* lr_exp_decay**(epoch - lr_rampup_epochs - lr_sustain_epochs)+ lr_min
return lr
return lrfn
blended_preds = pd.DataFrame()
blended_preds['id_seqpos'] = gru_preds['id_seqpos']
blended_preds['reactivity'] = gru_weight*gru_preds['reactivity'] + lstm_weight*lstm_preds['reactivity']
blended_preds['deg_Mg_pH10'] = gru_weight*gru_preds['deg_Mg_pH10'] + lstm_weight*lstm_preds['deg_Mg_pH10']
blended_preds['deg_pH10'] = gru_weight*gru_preds['deg_pH10'] + lstm_weight*lstm_preds['deg_pH10']
blended_preds['deg_Mg_50C'] = gru_weight*gru_preds['deg_Mg_50C'] + lstm_weight*lstm_preds['deg_Mg_50C']
blended_preds['deg_50C'] = gru_weight*gru_preds['deg_50C'] + lstm_weight*lstm_preds['deg_50C']<merge>
class TTACallback(Callback):
def __init__(self, test_data, score_thr):
self.test_data = test_data
self.score_thr = score_thr
self.test_pred = []
def on_epoch_end(self, epoch, logs=None):
if logs['val_categorical_accuracy'] > self.score_thr:
print('Run TTA...')
 self.test_pred.append(self.model.predict(self.test_data))
submission = sample_sub[['id_seqpos']].merge(blended_preds, on=['id_seqpos'])
 submission.head()<save_to_csv>
class SoftProbField(Layer):
def __init__(self, **kwargs):
super(SoftProbField, self ).__init__(**kwargs)
 def build(self, input_shape):
super(SoftProbField, self ).build(input_shape)
 def call(self, x):
h = x[:, 0]
s = x[:, 1]
r = x[:, 2]
m = s*r*(1-h)
 s = s*(1-h)*(1-m)
 r = r*(1-h)*(1-m)
 return tf.stack([h, s, r, m], axis=-1)
 def compute_output_shape(self, input_shape):
return(input_shape[0], 4 )
submission.to_csv(f'submission_new.csv', index=False)
 print('Submission saved' )<import_modules>
with strategy.scope() :
model = tf.keras.Sequential([
efn.EfficientNetB5(
 input_shape=(512, 512, 3),
weights='noisy-student',
include_top=False 
),
L.GlobalMaxPooling2D() ,
L.Dense(3**3),
L.LayerNormalization() ,
L.LeakyReLU(0.1),
L.Dropout(0.5),
L.Dense(3, activation='sigmoid'),
SoftProbField()
])
 model.compile(
 optimizer='adam',
loss = 'categorical_crossentropy',
metrics=['categorical_accuracy'] 
)
 model.summary()
from __future__ import print_function, absolute_import
import os
import sys
import time
import datetime
import argparse
import os.path as osp
import numpy as np
import random
from PIL import Image
import tqdm
import cv2
import csv
import math
import torchvision as tv
import torchvision
import torch.nn.functional as F
import torch.optim as optim
import torch
import torch.nn as nn
import torch.backends.cudnn as cudnn
from sklearn.metrics import f1_score
from torch.utils.data import DataLoader
from torch.autograd import Variable
from torch.optim import lr_scheduler
from tqdm import tqdm
from torch.utils.data import Dataset
import torchvision.transforms as transforms
from tensorboardX import SummaryWriter
import skimage.io<load_from_csv>
weights = model.get_weights()
name_file='.. /input/prostate-cancer-grade-assessment/test.csv'
data_dir = '.. /input/prostate-cancer-grade-assessment'
image_folder = os.path.join(data_dir, 'test_images')
 is_test = os.path.exists(image_folder)
 image_folder = image_folder if is_test else os.path.join(data_dir, 'train_images')
 csv_file=csv.reader(open(name_file,'r'))
content=[]
for line in csv_file:
print(line[0])
 content.append(line[0]+'.tiff')
 content=content[1:]
if not is_test:
content.append('002a4db09dad406c85505a00fb6f6144.tiff')
 content.append('007433133235efc27a39f11df6940829.tiff')
 content.append('0018ae58b01bdadc8e347995b69f99aa.tiff')
 content.append('0076bcb66e46fb485f5ba432b9a1fe8a.tiff')
 content.append('003d4dd6bd61221ebc0bfb9350db333f.tiff')
 content=content[3:]
print(content )<normalization>
IMAGE_SIZE = [512, 512]
def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):
rotation = math.pi * rotation / 180.
 shear = math.pi * shear / 180.
 c1 = tf.math.cos(rotation)
 s1 = tf.math.sin(rotation)
 one = tf.constant([1],dtype='float32')
 zero = tf.constant([0],dtype='float32')
 rotation_matrix = tf.reshape(tf.concat([c1,s1,zero, -s1,c1,zero, zero,zero,one],axis=0),[3,3])
 c2 = tf.math.cos(shear)
 s2 = tf.math.sin(shear)
 shear_matrix = tf.reshape(tf.concat([one,s2,zero, zero,c2,zero, zero,zero,one],axis=0),[3,3])
 zoom_matrix = tf.reshape(tf.concat([one/height_zoom,zero,zero, zero,one/width_zoom,zero, zero,zero,one],axis=0),[3,3])
 shift_matrix = tf.reshape(tf.concat([one,zero,height_shift, zero,one,width_shift, zero,zero,one],axis=0),[3,3])
 return K.dot(K.dot(rotation_matrix, shear_matrix), K.dot(zoom_matrix, shift_matrix))
def transform(image,label):
DIM = IMAGE_SIZE[0]
XDIM = DIM%2
rot = 45.* tf.random.normal([1],dtype='float32')
 shr = 32.* tf.random.normal([1],dtype='float32')
 h_zoom = 1.0 + tf.random.normal([1],dtype='float32')/5.
 w_zoom = 1.0 + tf.random.normal([1],dtype='float32')/5.
 h_shift = 32.* tf.random.normal([1],dtype='float32')
 w_shift = 32.* tf.random.normal([1],dtype='float32')
 m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift)
 x = tf.repeat(tf.range(DIM//2,-DIM//2,-1), DIM)
 y = tf.tile(tf.range(-DIM//2,DIM//2),[DIM])
 z = tf.ones([DIM*DIM],dtype='int32')
 idx = tf.stack([x,y,z])
 idx2 = K.dot(m,tf.cast(idx,dtype='float32'))
idx2 = K.cast(idx2,dtype='int32')
 idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)
 idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])
 d = tf.gather_nd(image,tf.transpose(idx3))
return tf.reshape(d,[DIM,DIM,3]),label
def gem(x, p=3, eps=1e-6):
return F.avg_pool2d(x.clamp(min=eps ).pow(p),(x.size(-2), x.size(-1)) ).pow(1./p)
 class GeM(nn.Module):
def __init__(self, p=3, eps=1e-6):
super(GeM,self ).__init__()
self.p = Parameter(torch.ones(1)*p)
 self.eps = eps
def forward(self, x):
return gem(x, p=self.p, eps=self.eps)
 def __repr__(self):
return self.__class__.__name__ + '(' + 'p=' + '{:.4f}'.format(self.p.data.tolist() [0])+ ', ' + 'eps=' + str(self.eps)+ ')'<categorify>
def get_datasets(tr_idx, vl_idx):
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( train_paths[tr_idx], train_labels[tr_idx])) 
.map(decode_image, num_parallel_calls=AUTO)
.cache() 
.map(data_augment, num_parallel_calls=AUTO)
.map(transform, num_parallel_calls=AUTO)
.repeat() 
.shuffle(512)
.batch(64)
.prefetch(AUTO)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( train_paths[vl_idx], train_labels[vl_idx])) 
.map(decode_image, num_parallel_calls=AUTO)
.batch(64)
.cache() 
.prefetch(AUTO)
)
 return train_dataset, valid_dataset
def get_tiles_combine(img,mode=0):
images = np.ones(( 1536, 1536, 3)) *255
h, w, c = img.shape
result_all=[]
pad_h =(256 - h % 256)% 256 +(( 256 * mode)// 2)
 pad_w =(256 - w % 256)% 256 +(( 256 * mode)// 2)
 img2 = np.pad(img,[[pad_h // 2, pad_h - pad_h // 2], [pad_w // 2,pad_w - pad_w//2], [0,0]], 'constant',constant_values=255)
 windows=[256,256,256,256,192,192,128]
x_start=0
for i in range(len(windows)) :
result = []
window_size=windows[i]
for x in range(( h+pad_h)//window_size):
for y in range(( w+pad_w)//window_size):
tile=img2[x*window_size:(x+1)*window_size,y*window_size:(y+1)*window_size]
result.append([x,y,tile.sum() ])
 result.sort(key=lambda ele:ele[2])
 result=result[:1536//window_size]
for y in range(min(1536//window_size,len(result))):
xx=result[y][0]
yy=result[y][1]
result_all.append([xx,yy])
 images[x_start:x_start+window_size,y*window_size:(y+1)*window_size]=\
img2[xx*window_size:(xx+1)*window_size,yy*window_size:(yy+1)*window_size].copy()
img2[xx*window_size:(xx+1)*window_size,yy*window_size:(yy+1)*window_size]=255
x_start=x_start+windows[i]
return images
class panda_dataset_combine(Dataset):

 def __init__(self, txt_path,transform=None):
imgs = []
for img in txt_path:
imgs.append(img)
 self.imgs = imgs
self.transform = transform
def __getitem__(self, index):
fn = self.imgs[index]
tif_name=fn
img_name=tif_name[:-5]
print(image_folder+'/'+tif_name)
 img=skimage.io.MultiImage(image_folder+'/'+tif_name)[1]
images=get_tiles_combine(img)
 images2=get_tiles_combine(img,mode=2)
 images=images.astype(np.float32)
 images=cv2.cvtColor(images,cv2.COLOR_BGR2RGB)
 images2=images2.astype(np.float32)
 images2=cv2.cvtColor(images2,cv2.COLOR_BGR2RGB)
 images3=cv2.flip(images,0)
 images4=cv2.flip(images2,0)
 images5=cv2.flip(images,1)
 images6=cv2.flip(images2,1)
 images = Image.fromarray(np.uint8(images))
images2 = Image.fromarray(np.uint8(images2))
images3 = Image.fromarray(np.uint8(images3))
images4 = Image.fromarray(np.uint8(images4))
images5 = Image.fromarray(np.uint8(images5))
images6 = Image.fromarray(np.uint8(images6))
if self.transform is not None:
img = self.transform(images)
 img2= self.transform(images2)
 img3= self.transform(images3)
 img4= self.transform(images4)
 img5= self.transform(images5)
 img6= self.transform(images6)
 return img_name,img,img2,img3,img4,img5,img6
def __len__(self):
return len(self.imgs )<categorify>
gkf = StratifiedKFold(n_splits=5)
 tta = TTACallback(test_dataset, 0.96)
 STEPS_PER_EPOCH = train_labels.shape[0] // 64
for i,(tr_idx, vl_idx)in enumerate(gkf.split(train_paths, train_labels.argmax(-1))):
print(f'Start {i} fold')
 tf.keras.backend.clear_session()
model.set_weights(weights)
 train_dataset, valid_dataset = get_datasets(tr_idx, vl_idx)
 lr_schedule = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=1)
 history = model.fit(
 train_dataset,
epochs=20,
callbacks=[lr_schedule, tta],
steps_per_epoch=STEPS_PER_EPOCH * 6,
validation_data=valid_dataset
)
<predict_on_test><EOS> 
probs = np.mean(tta.test_pred, axis=0)
 sub.loc[:, 'healthy':] = probs
sub.to_csv('submission.csv', index=False)
 sub.head()