Kaggle competition: Plant Pathology 2020 - FGVC7
<SOS> metric: MCAUC Kaggle data source: plant-pathology-2020-fgvc7<prepare_output>
df_submission4 = df_1.copy()
df_submission4['target'] = 0.3 * df_1['target'] + 0.1 * df_2b['target'] + 0.3 * df_3b['target'] + 0.3 * df_4['target']<save_to_csv>
!nvidia-smi
df_submission4.to_csv('submission4.csv', index=False)
 df_submission4<define_variables>
USE_COLAB, TRAIN_MODE, USE_TPU = 0, 1, 0
version_name = 'v16'
network_name = 'efficientnet_b4'
DEVICE = "TPU"
CFG = dict(
 net_count = 7,
batch_size = 32,
read_size = 512,
crop_size = 512,
net_size = 512,
LR_START = 0.000005,
LR_MAX = 0.000020,
LR_MIN = 0.000001,
LR_RAMPUP_EPOCHS = 10,
LR_SUSTAIN_EPOCHS = 2,
LR_EXP_DECAY = 0.7,
epochs = 25,
rot = 180.0,
shr = 2.0,
hzoom = 8.0,
wzoom = 8.0,
hshift = 8.0,
wshift = 8.0,
optimizer = 'adam',
label_smooth_fac = 0.05,
tta_steps = 25
)<install_modules>
if USE_TPU:
!curl https://raw.githubusercontent.com/pytorch/xla/master/contrib/scripts/env-setup.py -o pytorch-xla-env-setup.py
!python pytorch-xla-env-setup.py --apt-packages libomp5 libopenblas-dev
!pip install -q efficientnet<import_modules>
if not USE_COLAB:
print(os.listdir('.. /input/'))
PATH = '.. /input/plant-pathology-image-processing/'
SAVE_PATH = './'
MODEL_PATH = '.. /input/plant-pathology-pytorch-gpu-model/'
!git clone -q https://github.com/welkin-feng/ComputerVision.git
sys.path.append('./ComputerVision/')
 !git clone -q https://github.com/rwightman/pytorch-image-models.git
sys.path.append('./pytorch-image-models/')
 else:
drive.mount('/content/drive', force_remount=True)
 PATH = './drive/My Drive/Competition/plant-pathology-2020/plant-pathology-2020/'
SAVE_PATH = f"./drive/My Drive/Competition/plant-pathology-2020/{version_name}_{network_name}/"
MODEL_PATH = SAVE_PATH
if not os.path.isdir(SAVE_PATH):
os.mkdir(SAVE_PATH)
 !git clone -q https://github.com/welkin-feng/ComputerVision.git
sys.path.append('./ComputerVision/')
 !git clone -q https://github.com/rwightman/pytorch-image-models.git
sys.path.append('./pytorch-image-models/')
 print("PATH: ", os.listdir(PATH))
print("SAVE_PATH: ", os.listdir(SAVE_PATH))
if os.path.isdir(MODEL_PATH):
print('MODEL_PATH: ', os.listdir(MODEL_PATH))
DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 print(DEVICE)
 random_seed = 644
print(f'random_state: {random_seed}' )
random.seed(a=42)
<load_from_csv>
def seed_everything(seed):

 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
if USE_TPU:
seed_everything(random_seed )
BASEPATH = ".. /input/siim-isic-melanoma-classification"
df_train = pd.read_csv(os.path.join(BASEPATH, 'train.csv'))
df_test = pd.read_csv(os.path.join(BASEPATH, 'test.csv'))
df_sub = pd.read_csv(os.path.join(BASEPATH, 'sample_submission.csv'))
GCS_PATH = KaggleDatasets().get_gcs_path('melanoma-512x512')
 GCS_PATH_2019 = KaggleDatasets().get_gcs_path('isic2019-512x512')
 files_train_2019 = np.array(tf.io.gfile.glob(GCS_PATH_2019 + '/train*.tfrec'))
files_train_2020 = np.array(tf.io.gfile.glob(GCS_PATH + '/train*.tfrec'))
files_train = np.sort(np.concatenate(( files_train_2020, files_train_2019)))
 files_test = np.sort(np.array(tf.io.gfile.glob(GCS_PATH + '/test*.tfrec')) )<choose_model_class>
network_name = network_name
version_name = version_name
DEVICE = DEVICE
PATH = PATH
SAVE_PATH = SAVE_PATH
MODEL_PATH = MODEL_PATH
IMAGE_FILE_NAME = ['train_images_320x512.npy', 'test_images_320x512.npy']
IMG_SHAPE =(1365, 2048, 3)
 INPUT_IMG_SHAPE =(320, 512, 3)
 RGB_MEAN = np.array([0.40379888, 0.5128721, 0.31294255])
 RGB_STD = np.array([0.20503741, 0.18957737, 0.1883159])
 n_fold = 5
fold =(0, 1, 2, 3, 4)
 BATCH_SIZE = 16 if not USE_TPU else 64
TEST_BATCH_SIZE = 1
accumulation_steps = 1
loss_weights =(1, 1)
 learning_rate = 1e-3
lr_ratio = np.sqrt(0.1)
 reduce_lr_metric = ['loss', 'score', 'both'][0]
patience = 5
warm_up_steps = 1
warm_up_lr_factor = 0.1
num_classes = 4
n_epochs = 50
train_epochs = 50
resume = False
pretrained = not resume
if DEVICE == "TPU":
print("connecting to TPU...")
 try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
print("Could not connect to TPU")
 tpu = None
if tpu:
try:
print("initializing TPU...")
 tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 print("TPU initialized")
 except _:
print("failed to initialize TPU")
 else:
DEVICE = "GPU"
if DEVICE != "TPU":
print("Using default strategy for CPU and single GPU")
 strategy = tf.distribute.get_strategy()
if DEVICE == "GPU":
print("Num GPUs Available: ", len(tf.config.experimental.list_physical_devices('GPU')))
 AUTO = tf.data.experimental.AUTOTUNE
REPLICAS = strategy.num_replicas_in_sync
print(f'REPLICAS: {REPLICAS}' )<normalization>
df_train = pd.read_csv(PATH + 'train.csv')
 df_train['class'] = np.argmax(df_train.iloc[:, 1:].values, axis=1)
 skf = StratifiedKFold(n_fold, shuffle = True, random_state = 644)
 for i_fold,(train_idx, val_idx)in enumerate(skf.split(df_train, df_train['class'].values)) :
df_train.loc[val_idx, 'fold'] = i_fold
df_train['fold'] = df_train['fold'].astype(int)
 df_test = pd.read_csv(PATH + 'test.csv')
 submission = pd.read_csv(PATH + 'sample_submission.csv' )
def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):
rotation = math.pi * rotation / 180.
 shear = math.pi * shear / 180.
 def get_3x3_mat(lst):
return tf.reshape(tf.concat([lst],axis=0), [3,3])
 c1 = tf.math.cos(rotation)
 s1 = tf.math.sin(rotation)
 one = tf.constant([1],dtype='float32')
 zero = tf.constant([0],dtype='float32')
 rotation_matrix = get_3x3_mat([c1, s1, zero,
-s1, c1, zero,
zero, zero, one])
 c2 = tf.math.cos(shear)
 s2 = tf.math.sin(shear)
 shear_matrix = get_3x3_mat([one, s2, zero,
zero, c2, zero,
zero, zero, one])
 zoom_matrix = get_3x3_mat([one/height_zoom, zero, zero,
zero, one/width_zoom, zero,
zero, zero, one])
 shift_matrix = get_3x3_mat([one, zero, height_shift,
zero, one, width_shift,
zero, zero, one])
 return K.dot(K.dot(rotation_matrix, shear_matrix),
K.dot(zoom_matrix, shift_matrix))
def transform(image, cfg):
DIM = cfg["read_size"]
XDIM = DIM%2
rot = cfg['rot'] * tf.random.normal([1], dtype='float32')
 shr = cfg['shr'] * tf.random.normal([1], dtype='float32')
 h_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ cfg['hzoom']
w_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ cfg['wzoom']
h_shift = cfg['hshift'] * tf.random.normal([1], dtype='float32')
 w_shift = cfg['wshift'] * tf.random.normal([1], dtype='float32')
 m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift)
 x = tf.repeat(tf.range(DIM//2, -DIM//2,-1), DIM)
 y = tf.tile(tf.range(-DIM//2, DIM//2), [DIM])
 z = tf.ones([DIM*DIM], dtype='int32')
 idx = tf.stack([x,y,z])
 idx2 = K.dot(m, tf.cast(idx, dtype='float32'))
idx2 = K.cast(idx2, dtype='int32')
 idx2 = K.clip(idx2, -DIM//2+XDIM+1, DIM//2)
 idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])
 d = tf.gather_nd(image, tf.transpose(idx3))
return tf.reshape(d,[DIM, DIM,3] )<prepare_x_and_y>
for i_fold in range(5):
train_idx, valid_idx = np.where(( df_train['fold'] != i_fold)) [0], np.where(( df_train['fold'] == i_fold)) [0]
train_class = df_train['class'].iloc[train_idx].values.tolist()
val_class = df_train['class'].iloc[valid_idx].values.tolist()
train_class_count = {i:train_class.count(i)for i in range(4)}
val_class_count = {i:val_class.count(i)for i in range(4)}
print(f"i_fold {i_fold} train_class count {train_class_count} val_class_count {val_class_count}" )
def read_labeled_tfrecord(example):
tfrec_format = {
'image' : tf.io.FixedLenFeature([], tf.string),
'image_name' : tf.io.FixedLenFeature([], tf.string),
'patient_id' : tf.io.FixedLenFeature([], tf.int64),
'sex' : tf.io.FixedLenFeature([], tf.int64),
'age_approx' : tf.io.FixedLenFeature([], tf.int64),
'anatom_site_general_challenge': tf.io.FixedLenFeature([], tf.int64),
'diagnosis' : tf.io.FixedLenFeature([], tf.int64),
'target' : tf.io.FixedLenFeature([], tf.int64)
 }
example = tf.io.parse_single_example(example, tfrec_format)
 return(example['image'],(example['sex'], example['age_approx'], example['anatom_site_general_challenge']), example['target'])
 def read_unlabeled_tfrecord(example, return_image_name):
tfrec_format = {
'image' : tf.io.FixedLenFeature([], tf.string),
'image_name' : tf.io.FixedLenFeature([], tf.string),
'patient_id' : tf.io.FixedLenFeature([], tf.int64),
'sex' : tf.io.FixedLenFeature([], tf.int64),
'age_approx' : tf.io.FixedLenFeature([], tf.int64),
'anatom_site_general_challenge': tf.io.FixedLenFeature([], tf.int64)
 }
example = tf.io.parse_single_example(example, tfrec_format)
 return(example['image'],(example['sex'], example['age_approx'], example['anatom_site_general_challenge']),(example['image_name'] if return_image_name else 0))
def prepare_data(data, cfg=None, augment=True):
img = data[0]
img = tf.image.decode_jpeg(img, channels=3)
 img = tf.image.resize(img, [cfg['read_size'], cfg['read_size']])
 img = tf.cast(img, tf.float32)/ 255.0
if augment:
img = transform(img, cfg)
 img = tf.image.random_crop(img, [cfg['crop_size'], cfg['crop_size'], 3])
 img = tf.image.random_flip_left_right(img)
 img = tf.image.random_hue(img, 0.01)
 img = tf.image.random_saturation(img, 0.7, 1.3)
 img = tf.image.random_contrast(img, 0.8, 1.2)
 img = tf.image.random_brightness(img, 0.1)
 else:
img = tf.image.central_crop(img, cfg['crop_size'] / cfg['read_size'])
 img = tf.image.resize(img, [cfg['net_size'], cfg['net_size']])
 img = tf.reshape(img, [cfg['net_size'], cfg['net_size'], 3])
 sex_oh = tf.one_hot(data[1][0], 2)
 age_aprox = tf.dtypes.cast(tf.reshape(data[1][1], [1]), tf.float32)
 anatom_site_general_challenge = tf.one_hot(data[1][2], 7)
 dense = tf.concat([sex_oh, age_aprox, anatom_site_general_challenge], axis=0)
 return(img, dense)
 def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1))
for filename in filenames]
return np.sum(n )<categorify>
class PlantPathologyDataset(Dataset):
def __init__(self, csv, idx, mode, transform = None, data = None):
self.csv = csv.reset_index(drop = True)
 self.data = data
self.filepath_format = PATH + 'images/{}.jpg'
self.idx = np.asarray(idx ).astype('int')
 self.mode = mode
self.transform = transform
def __len__(self):
return self.idx.shape[0]
def __getitem__(self, index):
index = self.idx[index]
if self.data is not None:
image = self.data[index]
else:
img_name = self.csv['image_id'].iloc[index]
image = cv2.imread(self.filepath_format.format(img_name))
if image.shape != IMG_SHAPE:
image = cv2.rotate(image, rotateCode = cv2.ROTATE_90_CLOCKWISE)
 if image.shape != INPUT_IMG_SHAPE:
image = cv2.resize(image, INPUT_IMG_SHAPE[1::-1], interpolation = cv2.INTER_AREA)
 image = image.astype('uint8')
 image_origin = image.copy().astype('float32')
 image = self.transform(image ).astype('float32')if self.transform is not None else image.astype('float32')
 image, image_origin = np.rollaxis(image, 2, 0)/ 255, np.rollaxis(image_origin, 2, 0)/ 255
if self.mode == 'test':
return torch.tensor(image)
 else:
label = self.csv.iloc[index, 1:5].values.astype('float32')
 return torch.tensor(image), torch.tensor(image_origin), torch.tensor(label)
 def get_train_val_dataloader(i_fold, transforms_train, transforms_val):
train_idx, valid_idx = np.where(( df_train['fold'] != i_fold)) [0], np.where(( df_train['fold'] == i_fold)) [0]
train_data = np.load(PATH + IMAGE_FILE_NAME[0])if os.path.isfile(PATH + IMAGE_FILE_NAME[0])else None
dataset_train = PlantPathologyDataset(df_train, train_idx, 'train', transform=transforms_train, data = train_data)
 dataset_valid = PlantPathologyDataset(df_train, valid_idx, 'val', transform=transforms_val, data = train_data)
 batch_size, train_sampler, drop_last = BATCH_SIZE, RandomSampler(dataset_train), False
if USE_TPU and xm.xrt_world_size() > 1:
batch_size, drop_last = BATCH_SIZE // xm.xrt_world_size() , True
train_sampler = torch.utils.data.distributed.DistributedSampler(
 dataset_train, num_replicas=xm.xrt_world_size() , rank=xm.get_ordinal() , shuffle=True 
)
 train_loader = DataLoader(dataset_train, batch_size, sampler = train_sampler, drop_last = drop_last, num_workers=4)
 val_loader = DataLoader(dataset_valid, TEST_BATCH_SIZE, num_workers=4, pin_memory = True)
 return train_loader, val_loader
def get_test_dataloader() :
test_data = np.load(PATH + IMAGE_FILE_NAME[1])if os.path.isfile(PATH + IMAGE_FILE_NAME[1])else None
dataset_test = PlantPathologyDataset(df_test, np.arange(len(df_test)) , 'test', data = test_data)
 test_loader = DataLoader(dataset_test, TEST_BATCH_SIZE, sampler=None, num_workers=4)
 return test_loader
def get_dataset(files, cfg, augment = False, shuffle = False, repeat = False,
labeled=True, return_image_names=True):
ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO)
 ds = ds.cache()
if repeat:
ds = ds.repeat()
if shuffle:
ds = ds.shuffle(1024*8)
 opt = tf.data.Options()
opt.experimental_deterministic = False
ds = ds.with_options(opt)
 if labeled:
ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)
 else:
ds = ds.map(lambda example: read_unlabeled_tfrecord(example, return_image_names),
num_parallel_calls=AUTO)
 ds = ds.map(lambda img, dense, imgname_or_label:(prepare_data(( img, dense), augment=augment, cfg=cfg),
imgname_or_label),
num_parallel_calls=AUTO)
 ds = ds.batch(cfg['batch_size'] * REPLICAS)
 ds = ds.prefetch(AUTO)
 return ds<data_type_conversions>
def rand_bbox(img_shape, lam):
H, W = img_shape
cut_rat = np.sqrt(1.- lam)
 cut_w, cut_h = int(W * cut_rat), int(H * cut_rat)
 cx, cy = np.random.randint(cut_w // 4, W - cut_w // 4), np.random.randint(cut_h // 4, H - cut_h // 4)
 bbx1 = np.clip(cx - cut_w // 2, 0, W)
 bby1 = np.clip(cy - cut_h // 2, 0, H)
 bbx2 = np.clip(cx + cut_w // 2, 0, W)
 bby2 = np.clip(cy + cut_h // 2, 0, H)
 return bbx1, bby1, bbx2, bby2
class MixupBatch(object):
def __init__(self, mixup_prob, mixup_alpha, **kwargs):
self.prob = mixup_prob
self.alpha = mixup_alpha
def set_prob(self, epoch, max_epoch):
self.prob = min(1., epoch / max_epoch)
 def __call__(self, ori_img_batch, img_batch, label_batch):

 label_batch_mix, lam = None, 1
if self.alpha <= 0 or np.random.rand() > self.prob:
return img_batch, label_batch, label_batch_mix, lam
batch_size = ori_img_batch.shape[0]
lam = np.random.beta(self.alpha, self.alpha, batch_size)
 lam = np.maximum(1 - lam, lam)
 lam = torch.from_numpy(lam ).view(batch_size, 1, 1, 1 ).to(ori_img_batch)
 shuffled_idx = torch.randperm(batch_size)
 ori_img_batch = lam * ori_img_batch +(1 - lam)* ori_img_batch[shuffled_idx]
label_batch_mix = [label[shuffled_idx] for label in label_batch]
return ori_img_batch, label_batch, label_batch_mix, lam
class CutmixBatch(object):
def __init__(self, cutmix_prob, cutmix_alpha, **kwargs):
self.prob = cutmix_prob
self.alpha = cutmix_alpha
def set_prob(self, epoch, max_epoch):
self.prob = min(1., epoch / max_epoch)
 def __call__(self, ori_img_batch, img_batch, label_batch):

 label_batch_mix, lam = None, 1
if self.alpha <= 0 or np.random.rand() > self.prob:
return img_batch, label_batch, label_batch_mix, lam
batch_size = ori_img_batch.shape[0]
lam = np.random.beta(self.alpha, self.alpha, batch_size)
 lam = np.maximum(1 - lam, lam)
 lam = torch.from_numpy(lam ).view(batch_size, 1, 1, 1 ).to(ori_img_batch)
 shuffled_idx = torch.randperm(batch_size)
 y = ori_img_batch[shuffled_idx].clone().detach()
label_batch_mix = [label[shuffled_idx] for label in label_batch]
for i in range(batch_size):
bbx1, bby1, bbx2, bby2 = rand_bbox(ori_img_batch.shape[-2:], lam[i])
 ori_img_batch[i, :, bby1:bby2, bbx1:bbx2] = y[i, :, bby1:bby2, bbx1:bbx2]
lam[i] = 1 -(bbx2 - bbx1)*(bby2 - bby1)/ np.prod(ori_img_batch.shape[-2:])
 return ori_img_batch, label_batch, label_batch_mix, lam
class FMixBatch(object):
def __init__(self, fmix_prob, fmix_alpha=1, decay_power=3, max_soft=0.0, reformulate=False):
self.prob = fmix_prob
self.alpha = fmix_alpha
self.decay_power = decay_power
self.max_soft = max_soft
self.reformulate = reformulate
def set_prob(self, epoch, max_epoch):
self.prob = min(1., epoch / max_epoch)
 def __call__(self, ori_img_batch, img_batch, label_batch):

 label_batch_mix, lam = None, 1
if np.random.rand() > self.prob:
return img_batch, label_batch, label_batch_mix, lam
size = ori_img_batch.shape[-2:]
lam, mask = sample_mask(self.alpha, self.decay_power, size, self.max_soft, self.reformulate)
 mask = torch.from_numpy(mask ).to(ori_img_batch)
 shuffled_idx = torch.randperm(ori_img_batch.size(0))
ori_img_batch = mask * ori_img_batch +(1 - mask)* ori_img_batch[shuffled_idx]
label_batch_mix = [label[shuffled_idx] for label in label_batch]
return ori_img_batch, label_batch, label_batch_mix, lam
class MixBatch(object):
def __init__(self, transforms_dict = None, img_mean = np.zeros(1), img_std = np.ones(1), mixup_alpha = 0.4, cutmix_alpha = 1.0, **kwargs):
transforms_dict = transforms_dict or {}
self.mix_prob = transforms_dict.get('mix_prob', 0)
 probs = [transforms_dict.get('mixup_prob', 0), transforms_dict.get('cutmix_prob', 0),
transforms_dict.get('fmix_prob', 0), transforms_dict.get('grid_prob', 0)]
self.probs = np.cumsum(probs)
 cutout_ratio = transforms_dict.get('cutout_ratio',(0.05, 0.25))
self.mixup = MixupBatch(mixup_prob = 1, mixup_alpha = mixup_alpha)
 self.cutmix = CutmixBatch(cutmix_prob = 1, cutmix_alpha = cutmix_alpha)
 self.fmix = FMixBatch(fmix_prob = 1, fmix_alpha=1, decay_power=3, max_soft=0.0, reformulate=False)
 self.gridmask = [GridMaskBatch(num_grid =(3, 6), rotate = 15, mode = 0, prob = 1.) ,
GridMaskBatch(num_grid =(3, 6), rotate = 15, mode = 1, prob = 1.) ,
GridMaskBatch(num_grid =(3, 6), rotate = 15, mode = 2, prob = 1.) ,]
def set_prob(self, epoch, max_epoch = 15):
self.mixup.set_prob(epoch, max_epoch)
 self.cutmix.set_prob(epoch, max_epoch)
 self.fmix.set_prob(epoch, max_epoch)
 for gridmask in self.gridmask:
gridmask.set_prob(epoch, max_epoch)
 def __call__(self, ori_img_batch, img_batch, label_batch):
label_batch_mix, lam = None, 1
if np.random.rand() < self.mix_prob:
r = np.random.rand()
if r < self.probs[0]:
return self.mixup(ori_img_batch, img_batch, label_batch)
 elif r < self.probs[1]:
return self.cutmix(ori_img_batch, img_batch, label_batch)
 elif r < self.probs[2]:
return self.fmix(ori_img_batch, img_batch, label_batch)
 elif r < self.probs[3]:
img_batch = np.random.choice(self.gridmask )(ori_img_batch, img_batch)
 return img_batch, label_batch, label_batch_mix, lam
def show_dataset(thumb_size, cols, rows, ds):
mosaic = PIL.Image.new(mode='RGB', size=(thumb_size*cols +(cols-1),
thumb_size*rows +(rows-1)))
 for idx, data in enumerate(iter(ds)) :
img, target_or_imgid = data[0][0], data[1]
ix = idx % cols
iy = idx // cols
img = np.clip(img.numpy() * 255, 0, 255 ).astype(np.uint8)
 img = PIL.Image.fromarray(img)
 img = img.resize(( thumb_size, thumb_size), resample=PIL.Image.BILINEAR)
 mosaic.paste(img,(ix*thumb_size + ix,
iy*thumb_size + iy))
display(mosaic)
 ds = get_dataset(files_train, CFG ).unbatch().take(12*5)
 show_dataset(64, 12, 5, ds )<count_unique_values>
transforms_train, transforms_val = None, None
if TRAIN_MODE:
!pip install --upgrade -q albumentations
!pip install --upgrade -q imgaug
transforms_train = transforms.Compose([
lambda image: iaa.Sequential([
iaa.Fliplr(0.5),
iaa.Flipud(0.5),
] )(image=image),
lambda image: albumentations.OneOf([
albumentations.ShiftScaleRotate(shift_limit=0.2, scale_limit=0.2, rotate_limit=30, p=1),
] )(image=image)['image'],
lambda image: iaa.Sequential([
iaa.Sometimes(0.7, iaa.Cutout(nb_iterations=1, size=(0.6, 0.7), fill_mode="constant", cval=0))
] )(image=image),
])
 mix = None
len(df_train.diagnosis.unique() )<create_dataframe>
class RGBNorm(nn.Module):
IMAGENET_MEAN = torch.tensor(( 0.485, 0.456, 0.406)).view(1, 3, 1, 1)
 IMAGENET_STD = torch.tensor(( 0.229, 0.224, 0.225)).view(1, 3, 1, 1)
 def __init__(self, **kwargs):
super().__init__()
self.register_buffer('rgb_mean', torch.tensor(RGB_MEAN ).view(1,3,1,1))
self.register_buffer('rgb_std', torch.tensor(RGB_STD ).view(1,3,1,1))
def forward(self, x):
_, C, _, _ = x.shape
if C == 1:
x = x.repeat(( 1, 3, 1, 1))
x =(x - self.rgb_mean.to(x)) / self.rgb_std.to(x)
 return x
class Mish(nn.Module):
def __init__(self, *args, **kwagrs):
super(Mish, self ).__init__()
def forward(self,x):
return x.mul(F.softplus(x ).tanh())
 def gem(x, kernel_size, stride = None, p = 3, eps = 1e-6):
return F.avg_pool2d(x.clamp(min=eps ).pow(p), kernel_size, stride ).pow(1./p)
 class ArcMarginProduct(nn.Module):
def __init__(self, in_features, out_features):
super().__init__()
self.weight = nn.Parameter(torch.FloatTensor(out_features, in_features))
self.reset_parameters()
def reset_parameters(self):
stdv = 1./ math.sqrt(self.weight.size(1))
self.weight.data.uniform_(-stdv, stdv)
 def forward(self, features):
cosine = F.linear(F.normalize(features), F.normalize(self.weight))
return cosine
class Classifier(nn.Module):

 def __init__(self, backbone, num_classes, **kwargs):
super().__init__()
self.norm = RGBNorm()
self.backbone = backbone
self.gfc = nn.Sequential(*[nn.AdaptiveAvgPool2d(1),
nn.Flatten() ,
nn.Linear(backbone.out_channel, 2048),
Mish() ])
 self.dropouts = nn.ModuleList([nn.Dropout(0.5)for _ in range(5)])
 self.metric_classify = ArcMarginProduct(2048, num_classes)
 self.cls_head = nn.Linear(2048, num_classes)
 def forward(self, x):
x = self.norm(x)
 x = self.backbone(x)
 x = self.gfc(x)
 for i, dropout in enumerate(self.dropouts):
if i == 0:
out = self.cls_head(dropout(x))
else:
out += self.cls_head(dropout(x))
out /= len(self.dropouts)
 metric_output = self.metric_classify(x)
 return out, metric_output
def load_pretrained_model(model, model_path = '', url = '', skip =() , conversion =()):
if os.path.isfile(model_path):
state_dict = torch.load(model_path, map_location = 'cpu')
 if not USE_TPU or xm.is_master_ordinal() :
print('=> loading pretrained model {}'.format(model_path))
elif url != '':
state_dict = model_zoo.load_url(url, progress = False, map_location = 'cpu')
 if not USE_TPU or xm.is_master_ordinal() :
print('=> loading pretrained model {}'.format(url))
else:
return
conversion = np.array(conversion ).reshape(-1, 2)if len(conversion)else []
model_dict = model.state_dict()
pretrained_state_dict = {}
for ks in state_dict.keys() :
if ks in model_dict.keys() and all(s not in ks for s in skip):
km = ks
for _km, _ks in conversion:
if ks == _ks:
km = _km
break
pretrained_state_dict[km] = state_dict[ks]
if not USE_TPU or xm.is_master_ordinal() :
print(f"=> loading pretrained model weight length {len(pretrained_state_dict)} / total_state_dict {len(state_dict)} / total_model_dict {len(model_dict)}")
 model_dict.update(pretrained_state_dict)
 model.load_state_dict(model_dict, strict = False )
ds = tf.data.TFRecordDataset(files_train, num_parallel_reads=AUTO)
 ds = ds.take(1 ).cache().repeat()
ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)
 ds = ds.map(lambda img, dense, target:(prepare_data(( img, dense), cfg=CFG, augment=True), target),
num_parallel_calls=AUTO)
 ds = ds.take(12*5)
 ds = ds.prefetch(AUTO)
 show_dataset(64, 12, 5, ds )<create_dataframe>
def efficientnet_b4(pretrained = False, **kwargs):
if not USE_TPU or xm.is_master_ordinal() :
print('=> create `efficientnet_b4`')
 backbone = EfficientNet.from_name('efficientnet-b4')
 backbone._fc = None
backbone.out_channel = backbone.out_channels
backbone.forward = backbone.forward_features
if pretrained:
pretrained_file_name = ''
url = 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/efficientnet-b4-6ed6700e.pth'
load_pretrained_model(backbone, model_path = MODEL_PATH + pretrained_file_name, url = url,
skip =('.num_batches_tracked', '_fc.',))
model = Classifier(backbone, num_classes)
 return model
ds = get_dataset(files_test, CFG, augment=True, repeat=True,
labeled=False, return_image_names=False ).unbatch().take(12*5)
 show_dataset(64, 12, 5, ds )<choose_model_class>
def seresnext50_32x4d(pretrained = False, **kwargs):
if not USE_TPU or xm.is_master_ordinal() :
print('=> create `seresnext50_32x4d`')
 backbone_kwargs = dict(cardinality = 32, base_width = 4, act_layer = Mish, block_args = dict(attn_layer = 'se'))
backbone = ResNet(Bottleneck, [3, 4, 6, 3], in_chans = 3, **backbone_kwargs)
 backbone.fc = None
backbone.out_channel, backbone.forward = backbone.num_features, backbone.forward_features
if pretrained:
pretrained_file_name = 'gluon_seresnext50_32x4d-90cf2d6e.pth'
url = 'https://github.com/rwightman/pytorch-pretrained-gluonresnet/releases/download/v0.1/gluon_seresnext50_32x4d-90cf2d6e.pth'
load_pretrained_model(backbone, model_path = MODEL_PATH + pretrained_file_name, url = url,
skip =('.num_batches_tracked',))
model = Classifier(backbone, num_classes)
 return model
def get_lr_callback(cfg):
lr_start = cfg['LR_START']
lr_max = cfg['LR_MAX'] * strategy.num_replicas_in_sync
lr_min = cfg['LR_MIN']
lr_ramp_ep = cfg['LR_RAMPUP_EPOCHS']
lr_sus_ep = cfg['LR_SUSTAIN_EPOCHS']
lr_decay = cfg['LR_EXP_DECAY']
def lrfn(epoch):
if epoch < lr_ramp_ep:
lr =(lr_max - lr_start)/ lr_ramp_ep * epoch + lr_start
elif epoch < lr_ramp_ep + lr_sus_ep:
lr = lr_max
else:
lr =(lr_max - lr_min)* lr_decay**(epoch - lr_ramp_ep - lr_sus_ep)+ lr_min
return lr
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)
 return lr_callback<choose_model_class>
def cross_entropy(preds, trues, class_weights = 1.0, reduction = 'mean', **kwargs):
class_weights = torch.tensor(class_weights ).to(preds)
 ce_loss = -torch.sum(class_weights * trues * F.log_softmax(preds, dim = 1), dim = 1)
 if reduction == 'mean':
return ce_loss.mean()
elif reduction == 'sum':
return ce_loss.sum()
elif reduction == 'none':
return ce_loss
def focol_loss(preds, trues, gamma = 2, class_weights = 1, reduction = 'mean', **kwargs):
ce_loss = cross_entropy(preds, trues, class_weights=class_weights, reduction = 'none')
 alpha = torch.sum(trues *(1 - F.softmax(preds, dim = 1)) ** gamma, dim = 1)
 pos_loss = alpha * ce_loss
if reduction == 'mean':
return pos_loss.mean()
elif reduction == 'sum':
return pos_loss.sum()
elif reduction == 'none':
return pos_loss
def ohem_loss(preds, trues, ohem_rate = 1, class_weights = 1, reduction = 'mean', **kwargs):
ohem_rate = max(0, min(1, ohem_rate))
ce_loss = cross_entropy(preds, trues, class_weights, reduction = 'none')
 loss = torch.topk(ce_loss, int(ohem_rate * ce_loss.size(0)))[0] if 0 <= ohem_rate < 1 else ce_loss
if reduction == 'mean':
return torch.mean(loss)
 elif reduction == 'sum':
return torch.sum(loss)
 elif reduction == 'none':
return loss
class ArcFaceLoss(nn.Module):
def __init__(self, s=30.0, m=0.5, reduction='mean'):
super().__init__()
self.reduction = reduction
self.s = s
self.cos_m = math.cos(m)
 self.sin_m = math.sin(m)
 self.th = math.cos(math.pi - m)
 self.mm = math.sin(math.pi - m)* m
def forward(self, logits, labels):
logits = logits.float()
cosine = logits
sine = torch.sqrt(1.0 - torch.pow(cosine, 2))
phi = cosine * self.cos_m - sine * self.sin_m
phi = torch.where(cosine > self.th, phi, cosine - self.mm)
 output =(labels * phi)+(( 1.0 - labels)* cosine)
 output *= self.s
loss = cross_entropy(output, labels, reduction = self.reduction)
 return loss / 2
def criterion(logits, metric_logits, trues, is_val = False, lam = 1, **kwargs):
assert len(trues)== len(logits)== len(metric_logits)
 weights = loss_weights
class_weights =(1, 4, 1, 1)
 if not is_val:
loss_0 = cross_entropy(logits, trues.float() , class_weights=class_weights, reduction='mean')* lam
loss_metric = ArcFaceLoss(reduction='mean' )(metric_logits, trues.float())* lam
loss =(loss_0 * weights[0] + loss_metric * weights[1])/ sum(weights)
 else:
loss_0 = cross_entropy(logits, trues.float() , class_weights=class_weights, reduction='sum')* lam
loss_metric = ArcFaceLoss(reduction='sum' )(metric_logits, trues.float())* lam
loss =(loss_0 * weights[0] + loss_metric * weights[1])/ sum(weights)
 return loss,(loss_0.detach() , loss_metric.detach() )
def get_model(cfg):
model_input = tf.keras.Input(shape=(cfg['net_size'], cfg['net_size'], 3), name='imgIn')
 model_input_data = tf.keras.Input(shape=(10,), name='dataIn')
 dense = tf.keras.layers.Dense(256, activation='relu' )(model_input_data)
 dense = tf.keras.layers.BatchNormalization()(dense)
 dense = tf.keras.layers.Dropout(0.5 )(dense)
 dense = tf.keras.layers.Dense(64, activation='relu' )(dense)
 dense = tf.keras.layers.BatchNormalization()(dense)
 dense = tf.keras.layers.Dropout(0.5 )(dense)
 constructor = getattr(efn, f'EfficientNetB6')
 x = constructor(include_top=False, weights='imagenet',
input_shape=(cfg['net_size'], cfg['net_size'], 3),
pooling='avg' )(model_input)
 x = tf.keras.layers.Concatenate()([x, dense])
 x = tf.keras.layers.Dense(256, activation='relu' )(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Dropout(0.5 )(x)
 x = tf.keras.layers.Dense(1, activation='sigmoid' )(x)
 model = tf.keras.Model([model_input, model_input_data], x, name='aNetwork')
 model.summary()
return model<choose_model_class>
def get_score(submission, solution):
roc_score = roc_auc_score(solution, submission)
 predictions = np.argmax(submission, axis = 1)
 trues = np.argmax(solution, axis = 1)
 hard_idx = np.where(np.max(submission, axis = 1)<= 0.9)[0]
easy_idx = np.where(np.max(submission, axis = 1)> 0.9)[0]
score = np.mean(predictions == trues)
 hard_score = np.mean(predictions[hard_idx] == trues[hard_idx])if len(hard_idx)else 0
easy_score = np.mean(predictions[easy_idx] == trues[easy_idx])if len(easy_idx)else 0
return roc_score, score, hard_score, easy_score
def get_each_score(submission, solution):
predictions = np.argmax(submission, axis = 1)
 trues = np.argmax(solution, axis = 1)
 roc_scores, scores, hard_scores, easy_scores = [0] * num_classes, [0] * num_classes, [0] * num_classes, [0] * num_classes
for i in range(num_classes):
roc_scores[i] = roc_auc_score(solution.T[i], submission.T[i])
 pred, true = predictions[trues==i], trues[trues==i]
hard_idx = np.where(submission[trues==i][:, i] <= 0.9)[0]
easy_idx = np.where(submission[trues==i][:, i] > 0.9)[0]
scores[i] = np.mean(pred == true)
 hard_scores[i] = np.mean(pred[hard_idx] == true[hard_idx])if len(hard_idx)else 0
easy_scores[i] = np.mean(pred[easy_idx] == true[easy_idx])if len(easy_idx)else 0
roc_score, score, hard_score, easy_score = get_score(submission, solution)
 roc_scores.insert(0, roc_score)
 scores.insert(0, score)
 hard_scores.insert(0, hard_score)
 easy_scores.insert(0, easy_score)
 return roc_scores, scores, hard_scores, easy_scores
def show_running_time(text):
def decorator(func):
@functools.wraps(func)
 def wrapper(*args, **kw):
t0 = time.time()
res = func(*args, **kw)
 print(f"{text} time: {time.time() -t0:.1f} s")
 return res
return wrapper
return decorator
def clear_cache(func):
@functools.wraps(func)
 def wrapper(*args, **kw):
torch.cuda.empty_cache()
gc.collect()
return func(*args, **kw)
 return wrapper
@show_running_time('train')
 @clear_cache
def train_epoch(model, loader, device, optimizer, verbose = False):
model.train()
train_loss = []
optimizer.zero_grad()
acc_steps = accumulation_steps
steps = len(loader)// acc_steps * acc_steps
progress_bar = tqdmauto(loader)if(not USE_TPU or xm.is_master_ordinal())and verbose else None
for batch_idx,(img_batch, origin_img_batch, label_batch)in enumerate(loader):
label_batch_mix, lam = None, 1
if mix is not None:
img_batch, label_batch, label_batch_mix, lam = mix(img_batch, origin_img_batch, label_batch)
 img_batch, label_batch = img_batch.to(device), label_batch.to(device)
 label_batch_mix = None if label_batch_mix is None else label_batch_mix.to(device)
 lam = lam.to(device)if isinstance(lam, torch.Tensor)else lam
if steps is not None and batch_idx >= steps:
acc_steps = 1
logits, metric_logits = model(img_batch)
 loss, _ = criterion(logits, metric_logits, label_batch, is_val = False, lam = lam)
 if label_batch_mix is not None:
loss_mix = criterion(logits, metric_logits, label_batch_mix, is_val = False, lam = 1 - lam)
 loss = loss + loss_mix
loss = loss / acc_steps
loss.backward()
if(batch_idx + 1)% acc_steps == 0:
if USE_TPU:
xm.optimizer_step(optimizer, barrier = True)
 else:
optimizer.step()
optimizer.zero_grad()
loss_np = loss.detach().cpu().item() * acc_steps
train_loss.append(loss_np)
 if(not USE_TPU or xm.is_master_ordinal())and verbose and(batch_idx <= 10 or(batch_idx - 10)% verbose_step == 0):
progress_bar.set_postfix_str(f"loss: {loss_np:.4f}, smooth_loss: {np.mean(train_loss[-20:]):.4f}")
 progress_bar.update(1 if batch_idx <= 10 else 30)
 return np.asarray(train_loss ).mean()
@clear_cache
def val_epoch(model, loader, device):
model.eval()
val_loss, val_loss1, val_loss2 = 0, 0, 0
preds, metric_preds, trues = [], [], []
with torch.no_grad() :
for img_batch, origin_img_batch, label_batch in loader:
img_batch, label_batch = img_batch.to(device), label_batch.to(device)
 logits, metric_logits = model(img_batch)
 loss,(loss1, loss2)= criterion(logits, metric_logits, label_batch, is_val = True)
 val_loss += loss.detach().cpu().item()
val_loss1 += loss1.detach().cpu().item()
val_loss2 += loss2.detach().cpu().item()
preds.append(F.softmax(logits, dim = 1 ).cpu().numpy())
 metric_preds.append(metric_logits.cpu().numpy())
 trues.append(label_batch.cpu().numpy())
 preds = np.concatenate(preds)
 metric_preds = np.concatenate(metric_preds)
 trues = np.concatenate(trues)
 scores1 = get_each_score(preds, trues)
 scores2 = get_each_score(metric_preds, trues)
 val_result =(val_loss / len(trues), val_loss1 / len(trues), val_loss2 / len(trues), preds, metric_preds, trues, scores1, scores2)
 return val_result
def compile_new_model(cfg, old_model=None):
with strategy.scope() :
if(old_model is None):
model = get_model(cfg)
 else:
model = old_model
model.compile(
 optimizer = tf.keras.optimizers.Adam(learning_rate=0.1),
loss = tf.keras.losses.BinaryCrossentropy(label_smoothing = cfg['label_smooth_fac']),
metrics = [tf.keras.metrics.AUC(name='auc')])
 return model<load_from_csv>
def resume_model(model, optimizer = None, lr_scheduler = None, resume_file_path = '', device = 'cpu'):
other_param = {}
history, last_epoch, best_val_loss, best_val_score = None, None, None, None
if os.path.isfile(resume_file_path):
checkpoint = torch.load(resume_file_path, map_location = device)
 model.load_state_dict(checkpoint.pop('state_dict'), strict = False)
 print(f"load model success! last_epoch: {checkpoint['last_epoch']}, file_path: {resume_file_path}")
 if optimizer is not None and lr_scheduler is not None:
optimizer.load_state_dict(checkpoint['optimizer'])
 lr_scheduler.load_state_dict(checkpoint['lr_scheduler'])
 print(f"lr: {optimizer.state_dict() ['param_groups'][0]['lr']:.2e}")
 checkpoint.pop('optimizer', None)
 checkpoint.pop('lr_scheduler', None)
 other_param.update(checkpoint)
 print(f"cur_val_score: {other_param['cur_val_score']:.4f}, cur_val_loss: {other_param['cur_val_loss']:.4f}
" +
f"best_val_score: {other_param['best_val_score']:.4f}, best_val_loss: {other_param['best_val_loss']:.4f}
" +
f"best_epoch: {other_param['best_epoch']}, best_score_epoch: {other_param['best_score_epoch']}
")
 return model, optimizer, lr_scheduler, other_param
train = pd.read_csv('.. /input/siim-isic-melanoma-classification/train.csv')
 class_weights = class_weight.compute_class_weight('balanced',
np.unique(train.target),
train.target)
 class_weights = dict(enumerate(class_weights))<train_on_grid>
model_class = efficientnet_b4
file_path="/kaggle/working/effnet6_weights.best.hdf5"
checkpoint = ModelCheckpoint(file_path, monitor='auc', verbose=1, save_best_only=True, mode='max' )<create_dataframe>
@clear_cache
def train(i_fold, verbose = False):
device = DEVICE if not USE_TPU else xm.xla_device()
devices = [device] if not USE_TPU else [device]
lr = learning_rate if not USE_TPU else learning_rate * max(len(devices), xm.xrt_world_size())
 batch_size = BATCH_SIZE if not USE_TPU else BATCH_SIZE // max(len(devices), xm.xrt_world_size())
 filename = f"{version_name}_{network_name}_fold_{i_fold}.pth"
log_name = f"log_{version_name}_{network_name}_fold_{i_fold}.txt"
train_loader, val_loader = get_train_val_dataloader(i_fold, transforms_train, transforms_val)
 model = model_class(pretrained ).to(device)
 optimizer = torch.optim.AdamW(model.parameters() , lr)
 lr_scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, n_epochs, eta_min = 1e-6)
 train_param = {}
if resume:
if SAVE_PATH != MODEL_PATH and os.path.isfile(MODEL_PATH + log_name):
shutil.copy(MODEL_PATH + log_name, SAVE_PATH + log_name)
 model, optimizer, lr_scheduler, train_param = \
resume_model(model, optimizer, lr_scheduler, MODEL_PATH + filename, device)
 last_epoch = train_param.get('last_epoch', -1)
 best_val_loss, best_val_score = train_param.get('best_val_loss', 2**20), train_param.get('best_val_score', 0)
 best_epoch, best_score_epoch = train_param.get('best_epoch', -1), train_param.get('best_score_epoch', -1)
 history = train_param.get('history', pd.DataFrame())
 def save_model(model, optimizer, lr_scheduler, is_best = False, is_best_score = False):
model_state = {'state_dict': model.state_dict() ,
'last_epoch': last_epoch, 'history': history, 'i_fold': i_fold,
'cur_val_loss': val_loss, 'cur_val_score': val_score,
'best_val_loss': best_val_loss, 'best_val_score': best_val_score,
'best_epoch': best_epoch, 'best_score_epoch': best_score_epoch,
'optimizer': optimizer.state_dict() , 'lr_scheduler': lr_scheduler.state_dict() }
model_path = SAVE_PATH + f"{version_name}_{network_name}_fold_{i_fold}.pth"
if not USE_TPU:
torch.save(model_state, model_path)
 elif xm.is_master_ordinal() :
xm.save(model_state, model_path)
 else:
return
if is_best:
best_model_path = SAVE_PATH + f"{version_name}_{network_name}_fold_{i_fold}_best.pth"
shutil.copy(model_path, best_model_path)
 if is_best_score:
best_model_path = SAVE_PATH + f"{version_name}_{network_name}_fold_{i_fold}_best_score.pth"
shutil.copy(model_path, best_model_path)
 def show_attributes() :
attr =(f"train_set count: {len(train_loader.dataset)} val_set count: {len(val_loader.dataset)}
" +
f"save_path: {SAVE_PATH}
" + f"batch_size: {batch_size}, accumulation_steps: {accumulation_steps}
" +
f"current lr: {optimizer.param_groups[0]['lr']:.2e}, reduce_lr_metric: {reduce_lr_metric}
" +
f"best_val_score: {best_val_score:.5f}, best_val_loss: {best_val_loss:.5f}
" +
f"best_epoch: {best_epoch}, best_score_epoch: {best_score_epoch}
" +
f"last_epoch: {last_epoch}, train_epochs: {train_epochs}, n_epochs: {n_epochs}
")
 if hasattr(lr_scheduler, 'num_bad_epochs'):
attr += f"num_bad_epochs: {lr_scheduler.num_bad_epochs}, min_lr: {lr_scheduler.min_lrs[0]:.2e}, patience: {lr_scheduler.patience}
"
print('
' + attr + '
')
 return attr
try:
if not USE_TPU or xm.is_master_ordinal() :
attr = show_attributes()
if(not USE_TPU or xm.is_master_ordinal())and(not resume or not os.path.isfile(SAVE_PATH + log_name)) :
content =(f"
Training fold {i_fold}
filename: {filename}
train_data: {IMAGE_FILE_NAME[0]}
" +
f"train_img_size: {INPUT_IMG_SHAPE}
batch_size: {batch_size}
accumulation_steps: {accumulation_steps}
n_epochs: {n_epochs}
"+
f"model_class: {model_class.__name__}
optimizer: {optimizer}
lr_scheduler: {lr_scheduler}
")
 content += '
' + f"{time.ctime() } Training start
"
print(content)
 with open(SAVE_PATH + log_name, 'a')as appender:
appender.write(content)
 for epoch in range(last_epoch + 1, train_epochs):
cur_lr = optimizer.param_groups[0]['lr']
train_loss = train_epoch(model, train_loader, device, optimizer, verbose)
 val_result = val_epoch(model, val_loader, device)
 val_loss, val_loss1, val_loss2, preds, metric_preds, trues, scores1, scores2 = val_result
last_epoch, val_score = epoch, scores1[1][0]
is_best_loss, is_best_score = val_loss < best_val_loss, val_score > best_val_score
is_best =(is_best_loss or reduce_lr_metric is 'score')and(is_best_score or reduce_lr_metric is 'loss')
 best_val_loss, best_val_score = min(val_loss, best_val_loss), max(val_score, best_val_score)
 lr_scheduler.step()
content =(f"{time.ctime() } Epoch {epoch}, lr: {cur_lr:.2e}, " +
f"train loss: {train_loss:.5f}, val loss: {val_loss:.5f}, val loss1: {val_loss1:.4f}, val loss2: {val_loss2:.4f}, " +
f"roc_score1: {scores1[0][0]:.4f}, roc_score2: {scores2[0][0]:.4f}, " +
f"score1: {scores1[1][0]:.4f}, hard_score1: {scores1[2][0]:.4f}, easy_score1: {scores1[3][0]:.4f}, " +
f"score2: {scores2[1][0]:.4f}, hard_score2: {scores2[2][0]:.4f}, easy_score2: {scores2[3][0]:.4f}")
 if is_best:
best_epoch, content = epoch, content + " => best metric"
if is_best_score:
best_score_epoch, content = epoch, content + " => best score"
if not USE_TPU or xm.is_master_ordinal() :
print(content)
 with open(SAVE_PATH + log_name, 'a')as appender:
appender.write(content + '
')
 _h = pd.DataFrame({'train_loss': [train_loss], 'val_loss': [val_loss],
'roc_score1': [scores1[0][0]], 'roc_score2': [scores2[0][0]],
'score1': [scores1[1][0]], 'score2': [scores2[1][0]]})
 history = history.append(_h, ignore_index = True)
 save_model(model, optimizer, lr_scheduler, is_best, is_best_score)
 history.to_csv(SAVE_PATH + f"history_{version_name}_fold_{i_fold}.csv")
 content =(f"
best_val_score: {best_val_score:.5f}, best_val_loss: {best_val_loss:.5f}
" +
f"best_epoch: {best_epoch}, best_score_epoch: {best_score_epoch}
")
 with open(SAVE_PATH + log_name, 'a')as appender:
appender.write(content + '
')
 finally:
torch.cuda.empty_cache()
gc.collect()
if not USE_TPU or xm.is_master_ordinal() :
show_attributes()
ds_train = get_dataset(files_train, CFG, augment=True, shuffle=True, repeat=True)
 steps_train = count_data_items(files_train)/(CFG['batch_size'] * REPLICAS )<choose_model_class>
if TRAIN_MODE:
if DEVICE == torch.device('cuda'):
torch.backends.cudnn.benchmark = True
print('cudnn.benchmark = True')
 if USE_TPU:
torch.set_default_tensor_type('torch.FloatTensor')
 for i_fold in fold:
train(i_fold, verbose = False )
print("Buidling model...")
 model = compile_new_model(CFG )<train_model>
for i_fold in range(n_fold):
if os.path.isfile(SAVE_PATH + f"history_{version_name}_fold_{i_fold}.csv"):
history_file = SAVE_PATH + f"history_{version_name}_fold_{i_fold}.csv"
elif os.path.isfile(MODEL_PATH + f"history_{version_name}_fold_{i_fold}.csv"):
history_file = MODEL_PATH + f"history_{version_name}_fold_{i_fold}.csv"
else:
continue
print(f"show {history_file}")
 history = pd.read_csv(history_file)
 visualize_training_history(history, start_loc = 2 )
history = model.fit(ds_train,
verbose = 1,
steps_per_epoch = steps_train,
epochs = CFG['epochs'],
callbacks = [checkpoint, get_lr_callback(CFG)],
class_weight=class_weights )<load_pretrained>
def get_models(model_files, model_class):
device = DEVICE if not USE_TPU else xm.xla_device()
models = []
params = []
for model_f in model_files:
if os.path.isfile(SAVE_PATH + model_f):
resume_file_path = SAVE_PATH + model_f
elif os.path.isfile(MODEL_PATH + model_f):
resume_file_path = MODEL_PATH + model_f
else:
continue
model = model_class()
model, _, _, other_params = resume_model(model, resume_file_path = resume_file_path, device = 'cpu')
 model = model.to(device)
 model.eval()
other_params['sub_mode'] = ''
if 'best.' in model_f:
other_params['sub_mode'] = '_best'
elif 'best_score.' in model_f:
other_params['sub_mode'] = '_best_score'
models.append(model)
 params.append(other_params)
 return models, params
@show_running_time('test')
 def predict(model, loader):
device = DEVICE if not USE_TPU else xm.xla_device()
model.eval()
preds, metric_preds = [], []
with torch.no_grad() :
for img_batch in loader:
img_batch = img_batch.to(device)
 logits, metric_logits = model(img_batch)
 preds.append(F.softmax(logits, dim = 1 ).cpu().numpy())
 metric_preds.append(metric_logits.cpu().numpy())
 preds = np.concatenate(preds)
 metric_preds = np.concatenate(metric_preds)
 return preds, metric_preds
model = compile_new_model(CFG)
 model.load_weights(file_path )<predict_on_test>
def show_each_score(folds, model_files):
device = DEVICE if not USE_TPU else xm.xla_device()
models, params = get_models(model_files, model_class)
 cv = np.mean([p['cur_val_score'] for p in params])
 for i_fold, model, p in zip(folds, models, params):
_, val_loader = get_train_val_dataloader(i_fold, transforms_train, transforms_val)
 val_result = val_epoch(model, val_loader, device)
 val_loss, val_loss1, val_loss2, preds, metric_preds, trues, scores1, scores2 = val_result
print(f"
Fold {i_fold}, last_epoch: {p['last_epoch']}
val loss: {val_loss:.5f}, val loss1: {val_loss1:.4f}, val loss2: {val_loss2:.4f}")
 for i in range(5):
print(f"{i}, roc_score1: {scores1[0][i]:.4f}, roc_score2: {scores2[0][i]:.4f}, " +
f"score1: {scores1[1][i]:.4f}, hard_score1: {scores1[2][i]:.4f}, easy_score1: {scores1[3][i]:.4f}, " +
f"score2: {scores2[1][i]:.4f}, hard_score2: {scores2[2][i]:.4f}, easy_score2: {scores2[3][i]:.4f}" )
CFG['batch_size'] = 128
cnt_test = count_data_items(files_test)
 steps = cnt_test /(CFG['batch_size'] * REPLICAS)* CFG['tta_steps']
ds_testAug = get_dataset(files_test, CFG, augment=True, repeat=True,
labeled=False, return_image_names=False)
 probs = model.predict(ds_testAug, verbose=1, steps=steps)
 probs = np.stack(probs)
 probs = probs2 = probs[:cnt_test * CFG['tta_steps'],0]
probs = np.stack(np.split(probs, CFG['tta_steps'], axis=0), axis=1)
 probs = np.mean(probs, axis=1 )<create_dataframe>
model_files = [
f"{version_name}_{network_name}_fold_{0}_best.pth",
f"{version_name}_{network_name}_fold_{1}_best.pth",
f"{version_name}_{network_name}_fold_{2}_best.pth",
f"{version_name}_{network_name}_fold_{3}_best.pth",
f"{version_name}_{network_name}_fold_{4}_best.pth",
]
show_each_score(( 0,1,2,3,4), model_files )
ds = get_dataset(files_test, CFG, augment=False, repeat=False,
labeled=False, return_image_names=True)
 image_names = np.array([img_name.numpy().decode("utf-8")
 for img, img_name in iter(ds.unbatch())] )<save_to_csv>
@clear_cache
def predict_and_submission(test_loader, model_files):
submission1 = submission.copy()
submission2 = submission.copy()
n_fold_preds, n_fold_metric_preds = [], []
models, params = get_models(model_files, model_class)
 cv = np.mean([p['cur_val_score'] for p in params])
 sub_name = '_'.join([f"{p['i_fold']}_{p['last_epoch']+1}ep{p['sub_mode']}" for p in params])
 submission_name1 = f"{version_name}_{network_name}_{len(model_files)}_fold_submission1_cv{cv:.4f}_ifold_{sub_name}.csv"
submission_name2 = f"{version_name}_{network_name}_{len(model_files)}_fold_submission2_ifold_{sub_name}.csv"
with open(SAVE_PATH + f'log_{version_name}_{network_name}_submission.txt', 'a')as appender:
content =(f"
{time.ctime() }, submission file: {submission_name1}
" +
f"test_data: {IMAGE_FILE_NAME[0]}
test_img_size: {INPUT_IMG_SHAPE}
from models:
")
 for s, p in zip(model_files, params):
content += s + f", i_fold: {p['i_fold']}, train_epochs: {p['last_epoch']+1}ep, cur_val_score: {p['cur_val_score']:.5f}, cur_val_loss: {p['cur_val_loss']:.4f}, best_val_score: {p['best_val_score']:.5f}, best_val_loss: {p['best_val_loss']:.4f}
"
print(content)
 appender.write(content + '
')
 for model in models:
preds, metric_preds = predict(model, test_loader)
 n_fold_preds.append(preds)
 n_fold_metric_preds.append(metric_preds)
 preds = np.stack(n_fold_preds ).mean(axis = 0)
 submission1.iloc[:, 1:] = preds
submission1.to_csv(SAVE_PATH + submission_name1, index=False)
 display(submission1.tail(5))
submission = pd.DataFrame(dict(
 image_name = image_names,
target = probs[:]))
submission = submission.sort_values('image_name')
 submission.to_csv('submission_model.csv', index=False )<load_from_csv>
df_test = pd.read_csv(PATH + 'test.csv')
 submission = pd.read_csv(PATH + 'sample_submission.csv')
 test_loader = get_test_dataloader()
model_files = [
f"{version_name}_{network_name}_fold_{0}_best.pth",
f"{version_name}_{network_name}_fold_{1}_best.pth",
f"{version_name}_{network_name}_fold_{2}_best.pth",
f"{version_name}_{network_name}_fold_{3}_best.pth",
f"{version_name}_{network_name}_fold_{4}_best.pth",
]
predict_and_submission(test_loader, model_files )
train = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/train.csv')
 test = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/test.csv')
 sub = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/sample_submission.csv')
<define_variables>
model_files = [
f"{version_name}_{network_name}_fold_{0}.pth",
f"{version_name}_{network_name}_fold_{1}.pth",
f"{version_name}_{network_name}_fold_{2}.pth",
f"{version_name}_{network_name}_fold_{3}.pth",
f"{version_name}_{network_name}_fold_{4}.pth",
]
predict_and_submission(test_loader, model_files )