Kaggle competition: OSIC Pulmonary Fibrosis Progression
<SOS> metric: LaplaceLogLikelihood Kaggle data source: osic-pulmonary-fibrosis-progression<define_search_model>
warnings.filterwarnings("ignore" )
class enetv2(nn.Module):
def __init__(self, backbone, out_dim):
super(enetv2, self ).__init__()
self.enet = enet.EfficientNet.from_name(backbone)
 self.myfc = nn.Linear(self.enet._fc.in_features, out_dim)
 self.enet._fc = nn.Identity()
def extract(self, x):
return self.enet(x)
 def forward(self, x):
x = self.extract(x)
 x = self.myfc(x)
 return x
def load_models(model_files):
models = []
for model_f in model_files:
model_f = os.path.join(model_dir, model_f)
 backbone = 'efficientnet-b0'
model = enetv2(backbone, out_dim=5)
 model.load_state_dict(torch.load(model_f, map_location=lambda storage, loc: storage), strict=False)
 model.eval()
model.to(device)
 models.append(model)
 print(f'{model_f} loaded!')
 return models
model_files = [
'b0-stain-bb9_best_fold0.pth','b0-stain-964_best_fold1.pth','b0-stain-917_best_fold2.pth',
'b0-stain-c11_best_fold3.pth'
]
models = load_models(model_files )<categorify>
import tensorflow as tf
import tensorflow.keras.backend as K
import tensorflow.keras.layers as L
import tensorflow.keras.models as M
from sklearn.metrics import mean_absolute_error
from sklearn.model_selection import KFold
def sampleMeanStd(img):
img = img.astype("float32")
 b_ch=np.mean(img[:,:,0])
 g_ch=np.mean(img[:,:,1])
 r_ch=np.mean(img[:,:,2])
 img -= np.array(( b_ch,g_ch,r_ch))
b_ch=np.std(img[:,:,0])
 g_ch=np.std(img[:,:,1])
 r_ch=np.std(img[:,:,2])
 img /= np.array(( b_ch,g_ch,r_ch))
return img
def get_tiles(img, mode=0,forReference=False):
result = []
h, w, c = img.shape
pad_h =(tile_size - h % tile_size)% tile_size +(( tile_size * mode)// 2)
 pad_w =(tile_size - w % tile_size)% tile_size +(( tile_size * mode)// 2)
 img2 = np.pad(img,[[pad_h // 2, pad_h - pad_h // 2], [pad_w // 2,pad_w - pad_w//2], [0,0]], constant_values=255)
 img3 = img2.reshape(
 img2.shape[0] // tile_size,
tile_size,
img2.shape[1] // tile_size,
tile_size,
3 
)
 img3 = img3.transpose(0,2,1,3,4 ).reshape(-1, tile_size, tile_size,3)
 n_tiles_with_info =(img3.reshape(img3.shape[0],-1 ).sum(1)< tile_size ** 2 * 3 * 245 ).sum()
if len(img)< n_tiles:
img3 = np.pad(img3,[[0,N-len(img3)],[0,0],[0,0],[0,0]], constant_values=255)
 if forReference:
idxs = np.argsort(img3.reshape(img3.shape[0],-1 ).sum(-1)) [:n_tiles]
else:
idxs = np.argsort(img3.reshape(img3.shape[0],-1 ).sum(-1)) [:n_tiles_with_info]
img3 = img3[idxs]
for i in range(len(img3)) :
result.append({'img':img3[i], 'idx':i})
 return result, n_tiles_with_info
def getItemCustom(img_id,n_tiles=36):
tiff_file = os.path.join(train_folder, f'{img_id}.tiff')
 image = skimage.io.MultiImage(tiff_file)[1]
tiles, OK = get_tiles(image,forReference=True)
 idxes = list(range(n_tiles))
n_row_tiles = int(np.sqrt(n_tiles))
images = np.zeros(( image_size * n_row_tiles, image_size * n_row_tiles, 3))
for h in range(n_row_tiles):
for w in range(n_row_tiles):
i = h * n_row_tiles + w
if len(tiles)> idxes[i]:
this_img = tiles[idxes[i]]['img']
else:
this_img = np.ones(( self.image_size, self.image_size, 3)).astype(np.uint8)* 255
h1 = h * image_size
w1 = w * image_size
images[h1:h1+image_size, w1:w1+image_size] = this_img
return images 
<normalization>
df = pd.read_csv("/kaggle/input/osic-pulmonary-fibrosis-progression/train.csv")
 test = pd.read_csv("/kaggle/input/osic-pulmonary-fibrosis-progression/test.csv")
 sample = pd.read_csv("/kaggle/input/osic-pulmonary-fibrosis-progression/sample_submission.csv" )
reference_stain = getItemCustom('001c62abd11fa4b57bf7a6c603a11bb9')
 reference_stain = np.uint8(reference_stain)
 reference_stain = cv2.resize(reference_stain,(1024,1024))
reference_stain = staintools.LuminosityStandardizer.standardize(reference_stain)
 normalizer = staintools.StainNormalizer(method='vahadane')
 normalizer.fit(reference_stain)
 reference_stain = getItemCustom('008069b542b0439ed69b194674051964')
 reference_stain = np.uint8(reference_stain)
 reference_stain = cv2.resize(reference_stain,(1024,1024))
reference_stain = staintools.LuminosityStandardizer.standardize(reference_stain)
 normalizer2 = staintools.StainNormalizer(method='vahadane')
 normalizer2.fit(reference_stain)
 reference_stain = getItemCustom('0005f7aaab2800f6170c399693a96917')
 reference_stain = np.uint8(reference_stain)
 reference_stain = cv2.resize(reference_stain,(1024,1024))
reference_stain = staintools.LuminosityStandardizer.standardize(reference_stain)
 normalizer3 = staintools.StainNormalizer(method='vahadane')
 normalizer3.fit(reference_stain)
 reference_stain = getItemCustom('999a911f00a8647b3603859bf62c8c11')
 reference_stain = np.uint8(reference_stain)
 reference_stain = cv2.resize(reference_stain,(1024,1024))
reference_stain = staintools.LuminosityStandardizer.standardize(reference_stain)
 normalizer4 = staintools.StainNormalizer(method='vahadane')
 normalizer4.fit(reference_stain )<create_dataframe>
print(f"Out of 1549 entried there were only {df['Patient'].nunique() } unique patients" )
class PANDADataset(Dataset):
def __init__(self,
df,
image_size,
n_tiles=n_tiles,
tile_mode=0,
rand=False,
sub_imgs=False,
normalizer = None 
):
self.df = df.reset_index(drop=True)
 self.image_size = image_size
self.n_tiles = n_tiles
self.tile_mode = tile_mode
self.rand = rand
self.sub_imgs = sub_imgs
self.normalizer= normalizer
def __len__(self):
return self.df.shape[0]
def __getitem__(self, index):
row = self.df.iloc[index]
img_id = row.image_id
tiff_file = os.path.join(image_folder, f'{img_id}.tiff')
 image = skimage.io.MultiImage(tiff_file)[1]
tiles, tiles_count = get_tiles(image, self.tile_mode)
 idxes = list(range(tiles_count))
if tiles_count < 36:
idxes = list(range(self.n_tiles))
if tiles_count > 45:
idxes = list(range(self.n_tiles*2))
n_row_tiles = int(np.sqrt(self.n_tiles))
images = np.zeros(( image_size * n_row_tiles, image_size * n_row_tiles, 3))
for h in range(n_row_tiles):
for w in range(n_row_tiles):
i = h * n_row_tiles + w
if len(tiles)> idxes[i]:
this_img = tiles[idxes[i]]['img']
else:
this_img = np.ones(( self.image_size, self.image_size, 3)).astype(np.uint8)* 255
h1 = h * image_size
w1 = w * image_size
images[h1:h1+image_size, w1:w1+image_size] = this_img
if tiles_count > 45:
images2 = np.zeros(( image_size * n_row_tiles, image_size * n_row_tiles, 3))
for h in range(n_row_tiles):
for w in range(n_row_tiles):
i = h * n_row_tiles + w
if len(tiles)> idxes[i+36]:
this_img = tiles[idxes[i+36]]['img']
else:
this_img = np.ones(( self.image_size, self.image_size, 3)).astype(np.uint8)* 255
h1 = h * image_size
w1 = w * image_size
images2[h1:h1+image_size, w1:w1+image_size] = this_img
else:
images2 = np.copy(images)
 image_to_transform = np.copy(images)
 image_to_transform = np.uint8(image_to_transform)
 image_to_transform = cv2.resize(image_to_transform,(1024,1024))
try:
image_to_transform = staintools.LuminosityStandardizer.standardize(image_to_transform)
 normalized_image = self.normalizer.transform(image_to_transform)
 normalized_image = cv2.resize(normalized_image,(1536,1536))
images = normalized_image
except Exception as e:
print(e)
 images = 255 - images
images = images.astype(np.float32)
 images /= 255
images = images.transpose(2, 0, 1)
 if tiles_count > 45:
image_to_transform = np.copy(images2)
 image_to_transform = np.uint8(image_to_transform)
 image_to_transform = cv2.resize(image_to_transform,(1024,1024))
try:
image_to_transform = staintools.LuminosityStandardizer.standardize(image_to_transform)
 normalized_image = self.normalizer.transform(image_to_transform)
 normalized_image = cv2.resize(normalized_image,(1536,1536))
images2 = normalized_image
except Exception as e:
print(e)
 images2 = 255 - images2
images2 = images2.astype(np.float32)
 images2 /= 255
images2 = images2.transpose(2, 0, 1)
 else:
images2 = 255 - images2
images2 = images2.astype(np.float32)
 images2 /= 255
images2 = images2.transpose(2, 0, 1)
 return torch.tensor([images,images2])
<create_dataframe>
print("Since the data has dupilcated values let's recheck by dropping duplicates")
 print()
print("
print()
print(df[['Patient','Sex','SmokingStatus']].drop_duplicates() ['Sex'].value_counts())
 print()
print("
print()
print(df[['Patient','Sex','SmokingStatus']].drop_duplicates() ['SmokingStatus'].value_counts() )
dataset = PANDADataset(df, image_size, n_tiles, tile_mode=0,normalizer=normalizer)
 loader = DataLoader(dataset, batch_size=batch_size, num_workers=num_workers, shuffle=False)
 dataset2 = PANDADataset(df, image_size, n_tiles, tile_mode=0,normalizer=normalizer2)
 loader2 = DataLoader(dataset2, batch_size=batch_size, num_workers=num_workers, shuffle=False)
 dataset3 = PANDADataset(df, image_size, n_tiles, tile_mode=0,normalizer=normalizer3)
 loader3 = DataLoader(dataset3, batch_size=batch_size, num_workers=num_workers, shuffle=False)
 dataset4 = PANDADataset(df, image_size, n_tiles, tile_mode=0,normalizer=normalizer4)
 loader4 = DataLoader(dataset4, batch_size=batch_size, num_workers=num_workers, shuffle=False )<import_modules>
df['Photo count'] = 0
names = df['Patient'].unique()
for name in names:
file = '/kaggle/input/osic-pulmonary-fibrosis-progression/train/'+name
df.loc[df['Patient']==name,'Photo count'] = len(os.listdir(file))
data = df.groupby(by="Patient")["Photo count"].first().reset_index(drop=False)
 data = data.sort_values(['Photo count'] ).reset_index(drop=True)
 data['Photo count'].describe()
from time import time<normalization>
def seed_everything(seed=2020):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 seed_everything(42 )
start_time = time()
LOGITS = []
LOGITS2 = []
LOGITS3 = []
LOGITS4 = []
with torch.no_grad() :
for data in tqdm(loader):
all_logits = []
for i in range(data.shape[0]):
img1,img2 = data[i][0],data[i][1]
logits = models[0](data[i].to(device))
logits = torch.mean(logits,dim=0)
 all_logits.append(logits)
 logits = torch.stack(all_logits)
 LOGITS.append(logits)
 with torch.no_grad() :
for data in tqdm(loader2):
all_logits = []
for i in range(data.shape[0]):
img1,img2 = data[i][0],data[i][1]
logits = models[1](data[i].to(device))
logits = torch.mean(logits,dim=0)
 all_logits.append(logits)
 logits = torch.stack(all_logits)
 LOGITS2.append(logits)
 with torch.no_grad() :
for data in tqdm(loader3):
all_logits = []
for i in range(data.shape[0]):
img1,img2 = data[i][0],data[i][1]
logits = models[2](data[i].to(device))
logits = torch.mean(logits,dim=0)
 all_logits.append(logits)
 logits = torch.stack(all_logits)
 LOGITS3.append(logits)
 with torch.no_grad() :
for data in tqdm(loader4):
all_logits = []
for i in range(data.shape[0]):
img1,img2 = data[i][0],data[i][1]
logits = models[3](data[i].to(device))
logits = torch.mean(logits,dim=0)
 all_logits.append(logits)
 logits = torch.stack(all_logits)
 LOGITS4.append(logits)
 LOGITS_all =(torch.cat(LOGITS ).sigmoid().cpu() + torch.cat(LOGITS2 ).sigmoid().cpu() + torch.cat(LOGITS3 ).sigmoid().cpu() + torch.cat(LOGITS4 ).sigmoid().cpu())/ 4
PREDS = LOGITS_all.sum(1 ).round().numpy()
df['isup_grade'] = PREDS.astype(int)
 df[['image_id', 'isup_grade']].to_csv('submission.csv', index=False)
 print('Total time taken',time() -start_time)
 print(df.head())
 print()
print(df.isup_grade.value_counts() )<install_modules>
df.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'] )
!pip install /kaggle/input/pretrainedmodels
!pip install /kaggle/input/pytorchimagemodels120
!pip install /kaggle/input/efficientnet
!pip install /kaggle/input/pytorch-lightning-081-whl/pytorch_lightning-0.8.1-py3-none-any.whl
!pip install /kaggle/input/warmup
!pip install /kaggle/input/albumentations
!pip install /kaggle/input/imagecodecs/imagecodecs-2020.5.30-cp37-cp37m-manylinux2014_x86_64.whl
!pip install /kaggle/input/tifffile/tifffile-2020.6.3-py3-none-any.whl
!pip install /kaggle/input/panda-code-2
!mkdir -p /root/.cache/torch/checkpoints/
!cp /kaggle/input/efficientnetb0355c32eb/efficientnet-b0-355c32eb.pth /root/.cache/torch/checkpoints
!cp /kaggle/input/my-efficientnetb008094119/my_efficientnet-b0-08094119.pth /root/.cache/torch/checkpoints<define_variables>
sample['Patient'] = sample['Patient_Week'].apply(lambda x:x.split('_')[0])
 sample['Weeks'] = sample['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sample = sample[['Patient','Weeks','Confidence','Patient_Week']]
sample = sample.merge(test.drop('Weeks', axis=1), on="Patient" )
def load_models(exp_names):
models = []
for exp_name in exp_names:
model_dir = Path(".. /input/panda-code-2/trained_models")/ exp_name
nr_checkpoints = 1
model_path = get_ckpt_path_from_dir(model_dir, nr=nr_checkpoints)
 model_pathes = model_path if type(model_path)is list else [model_path]
for model_path in model_pathes:
print(f"model_path: {model_path}")
 model = ClassificationModel.load_from_checkpoint(str(model_path))
model.eval()
model.to(device)
 models.append(model)
 return models
exp_names_256 = [f"1um_bhM_reg2_ep25_mfRP_RgIcBl_ts256_se2222_f{f}" for f in range(5)]
exp_names_512 = [f"1um_bhM_cm20_mrK_reg2_RgIcBl_f{f}" for f in range(5)]
data_dir = Path('.. /input/prostate-cancer-grade-assessment')
 df_train = pd.read_csv(data_dir / 'train.csv')
 df_test = pd.read_csv(data_dir / 'test.csv')
 image_folder = data_dir / 'test_images'
is_test = os.path.exists(image_folder)
 image_folder = image_folder if is_test else data_dir / 'train_images'
df = df_test if is_test else df_train.loc[:5]
print(f"is_test: {is_test}")
 input_type = "tiff_1um"
batch_size = 2
num_workers = 2
tta_nr = 1
tile_mode = 0
head_type = "normal"
device = torch.device('cuda')
 models_256 = load_models(exp_names_256)
 models_512 = load_models(exp_names_512)
 normalize = False
if tta_nr > 1:
tfs_tile_train, tfs_all_train, _, _ = get_transforms_classification(False, False, False,
False, False, False,
prob=0.5, intensity="mid", normalize=normalize)
 else:
tfs_tile_train, tfs_all_train = None, None
dataset_256 = TileDataset(image_folder, df, 256, 256, 144, tile_mode=tile_mode,
transform_tile=None, transform_all=None, invert=False,
inference=True, input_type=input_type, overwrite_image_folder=False, tta_nr=tta_nr,
normalize=normalize)
 loader_256 = DataLoaderPytorch(dataset_256, batch_size=batch_size, num_workers=num_workers, shuffle=False)
 dataset_512 = TileDataset(image_folder, df, 512, 512, 36, tile_mode=tile_mode,
transform_tile=None, transform_all=None, invert=False,
inference=True, input_type=input_type, overwrite_image_folder=False, tta_nr=tta_nr,
normalize=normalize)
 loader_512 = DataLoaderPytorch(dataset_512, batch_size=batch_size, num_workers=num_workers, shuffle=False)
 logits = []
with torch.no_grad() :
for idx, data_both in enumerate(zip(loader_256, loader_512)) :
data_256 = data_both[0]
data_512 = data_both[1]
logits_batch_ens = []
for model in models_256:
data = data_256
for tta_idx in range(tta_nr):
if tta_nr > 1:
data_tta = data["data"][tta_idx].to(device)
 else:
data_tta = data["data"].to(device)
 data_raw = model(data_tta)
 if head_type == "hydra":
data_raw = data_raw[0]
logits_batch = data_raw.sigmoid().cpu().numpy()
logits_batch_ens.append(logits_batch)
 for model in models_512:
data = data_512
for tta_idx in range(tta_nr):
if tta_nr > 1:
data_tta = data["data"][tta_idx].to(device)
 else:
data_tta = data["data"].to(device)
 data_raw = model(data_tta)
 if head_type == "hydra":
data_raw = data_raw[0]
logits_batch = data_raw.sigmoid().cpu().numpy()
logits_batch_ens.append(logits_batch)
 logits_batch_ens = np.array(logits_batch_ens)
 logits_batch_ens = logits_batch_ens.mean(axis=0)
 logits.append(logits_batch_ens)
 logits = np.concatenate(logits, axis=0)
 preds = logits.sum(axis=1 ).round()
df['isup_grade'] = preds.astype(int)
 df[['image_id', 'isup_grade']].to_csv('submission.csv', index=False)
 print(df.head())
 print()
print(df.isup_grade.value_counts())
<define_variables>
df['WHERE'] = 'train'
test['WHERE'] = 'val'
sample['WHERE'] = 'test'
data = df.append([test, sample] )
DEBUG = False<define_variables>
data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min' )
sys.path = [
'.. /input/efficientnet-pytorch/EfficientNet-PyTorch/EfficientNet-PyTorch-master',
] + sys.path<import_modules>
base = data.loc[data.Weeks == data.min_week]
base = base[['Patient','FVC']].copy()
base.columns = ['Patient','min_FVC']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base.drop('nb', axis=1, inplace=True )
import skimage.io
import numpy as np
import pandas as pd
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.utils.data import DataLoader, Dataset
from efficientnet_pytorch import model as enet
import matplotlib.pyplot as plt
from tqdm import tqdm_notebook as tqdm 
<load_from_csv>
data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base
gc.collect()
data_dir = '.. /input/prostate-cancer-grade-assessment'
df_train = pd.read_csv(os.path.join(data_dir, 'train.csv'))
df_test = pd.read_csv(os.path.join(data_dir, 'test.csv'))
df_sub = pd.read_csv(os.path.join(data_dir, 'sample_submission.csv'))
image_folder = os.path.join(data_dir, 'test_images')
 is_test = os.path.exists(image_folder)
 image_folder = image_folder if is_test else os.path.join(data_dir, 'train_images')
 df = df_test if is_test else df_train.loc[:100]
n_tiles = 36
tile_size = 256
image_size = 256
batch_size = 4
num_workers = 4
device = torch.device('cuda')
 print(image_folder )<define_search_model>
data[['Sex_Male','SmokingStatus_Ex-smoker','SmokingStatus_Never smoked']] = pd.get_dummies(data[['Sex','SmokingStatus']],drop_first=True)
 features = ['Percent','Age','min_FVC','base_week']
class enetv2(nn.Module):
def __init__(self, backbone, out_dim):
super(enetv2, self ).__init__()
self.enet = enet.EfficientNet.from_name(backbone)
 self.myfc = nn.Linear(self.enet._fc.in_features, out_dim)
 self.enet._fc = nn.Identity()
def extract(self, x):
return self.enet(x)
 def forward(self, x):
x = self.extract(x)
 x = self.myfc(x)
 return x
def load_models(model_files):
models = []
for model_dict in model_files:
model_f = model_dict['file_name']
backbone = model_dict['model']
model = enetv2(backbone, out_dim=5)
 model.load_state_dict(torch.load(model_f, map_location=lambda storage, loc: storage), strict=True)
 model.eval()
model.to(device)
 models.append(model)
 print(f'{model_f} loaded!')
 return models 
<load_pretrained>
for i in features:
scaler = MinMaxScaler()
data[i] = scaler.fit_transform(data[[i]] )
model_files = [
{'file_name':'.. /input/panda-public-models/cls_effnet_b0_Rand36r36tiles256_big_bce_lr0.3_augx2_30epo_model_fold0.pth','model':'efficientnet-b0','n_tiles':36},
{'file_name':'.. /input/panda-enet-b1-model/enet_8ep_fold0.pth','model':'efficientnet-b1','n_tiles':36}
]
models = load_models(model_files )<categorify>
df = data.loc[data.WHERE=='train'].drop('WHERE',axis=1)
 test = data.loc[data.WHERE=='val'].drop('WHERE',axis=1)
 sample = data.loc[data.WHERE=='test'].drop('WHERE',axis=1 )
def get_tiles(img, mode=0):
result = []
h, w, c = img.shape
pad_h =(tile_size - h % tile_size)% tile_size +(( tile_size * mode)// 2)
 pad_w =(tile_size - w % tile_size)% tile_size +(( tile_size * mode)// 2)
 img2 = np.pad(img,[[pad_h // 2, pad_h - pad_h // 2], [pad_w // 2,pad_w - pad_w//2], [0,0]], constant_values=255)
 img3 = img2.reshape(
 img2.shape[0] // tile_size,
tile_size,
img2.shape[1] // tile_size,
tile_size,
3 
)
 img3 = img3.transpose(0,2,1,3,4 ).reshape(-1, tile_size, tile_size,3)
 n_tiles_with_info =(img3.reshape(img3.shape[0],-1 ).sum(1)< tile_size ** 2 * 3 * 255 ).sum()
if len(img)< n_tiles:
img3 = np.pad(img3,[[0,N-len(img3)],[0,0],[0,0],[0,0]], constant_values=255)
 idxs = np.argsort(img3.reshape(img3.shape[0],-1 ).sum(-1)) [:n_tiles]
img3 = img3[idxs]
for i in range(len(img3)) :
result.append({'img':img3[i], 'idx':i})
 return result, n_tiles_with_info >= n_tiles
class PANDADataset(Dataset):
def __init__(self,
df,
image_size,
n_tiles=n_tiles,
tile_mode=0,
rand=True,
sub_imgs=False 
):
self.df = df.reset_index(drop=True)
 self.image_size = image_size
self.n_tiles = n_tiles
self.tile_mode = tile_mode
self.rand = rand
self.sub_imgs = sub_imgs
def __len__(self):
return self.df.shape[0]
def __getitem__(self, index):
row = self.df.iloc[index]
img_id = row.image_id
tiff_file = os.path.join(image_folder, f'{img_id}.tiff')
 image = skimage.io.MultiImage(tiff_file)[1]
tiles, OK = get_tiles(image, self.tile_mode)
 if self.rand:
idxes = np.random.choice(list(range(self.n_tiles)) , self.n_tiles, replace=False)
 else:
idxes = list(range(self.n_tiles))
idxes = np.asarray(idxes)+ self.n_tiles if self.sub_imgs else idxes
n_row_tiles = int(np.sqrt(self.n_tiles))
images = np.zeros(( image_size * n_row_tiles, image_size * n_row_tiles, 3))
for h in range(n_row_tiles):
for w in range(n_row_tiles):
i = h * n_row_tiles + w
if len(tiles)> idxes[i]:
this_img = tiles[idxes[i]]['img']
else:
this_img = np.ones(( self.image_size, self.image_size, 3)).astype(np.uint8)* 255
this_img = 255 - this_img
h1 = h * image_size
w1 = w * image_size
images[h1:h1+image_size, w1:w1+image_size] = this_img
images = images.astype(np.float32)
 images /= 255
images = images.transpose(2, 0, 1)
 return torch.tensor(images)
<create_dataframe>
features += ['Sex_Male','SmokingStatus_Ex-smoker','SmokingStatus_Never smoked']
LOGITS_FINAL = []
for i, model in enumerate(models):
LOGITS = []
LOGITS2 = []
n_tiles = model_files[i]['n_tiles']
dataset = PANDADataset(df, image_size, n_tiles, 0)
 loader = DataLoader(dataset, batch_size=batch_size, num_workers=num_workers, shuffle=False)
 dataset2 = PANDADataset(df, image_size, n_tiles, 2)
 loader2 = DataLoader(dataset2, batch_size=batch_size, num_workers=num_workers, shuffle=False)
 with torch.no_grad() :
for data in tqdm(loader):
data = data.to(device)
 logits = model(data)
 LOGITS.append(logits)
 for data in tqdm(loader2):
data = data.to(device)
 logits = model(data)
 LOGITS2.append(logits)
 LOGITS_TMP =(torch.cat(LOGITS ).sigmoid().cpu() + torch.cat(LOGITS2 ).sigmoid().cpu())/ 2
LOGITS_FINAL.append(LOGITS_TMP)
 model_num = len(LOGITS_FINAL)
 PREDS = 0
for ans in LOGITS_FINAL:
PREDS += ans/model_num
PREDS = PREDS.sum(1 ).round().numpy()
df['isup_grade'] = PREDS.astype(int)
 df[['image_id', 'isup_grade']].to_csv('submission.csv', index=False)
 print(df.head())
 print()
print(df.isup_grade.value_counts() )<define_variables>
X = df[features].values
y = df['FVC'].values
test_ = sample[features].values
y = y.astype('float64' )
DEBUG = False<define_variables>
nh = X.shape[1]
pe = np.zeros(( test_.shape[0], 3))
pred = np.zeros(( X.shape[0], 3))
sys.path = [
'.. /input/efficientnet-pytorch/EfficientNet-PyTorch/EfficientNet-PyTorch-master',
] + sys.path<import_modules>
C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype="float32")
 def score(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = tf.maximum(sigma, C1)
 delta = tf.abs(tf.dtypes.cast(y_true[:, 0], tf.float32)- tf.dtypes.cast(fvc_pred, tf.float32))
delta = tf.minimum(delta, C2)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
metric =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)
 return K.mean(metric)
 def qloss(y_true, y_pred):
qs = [0.2, 0.50, 0.8]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true - y_pred
e = tf.dtypes.cast(y_true,tf.float32)- tf.dtypes.cast(y_pred,tf.float32)
 v = tf.maximum(q*e,(q-1)*e)
 return K.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss
def make_model(nh):
z = L.Input(( nh,), name="Patient")
 x = L.Dense(100, activation="relu", name="d1" )(z)
 x = L.Dense(100, activation="relu", name="d2" )(x)
 p1 = L.Dense(3, activation="linear", name="p1" )(x)
 p2 = L.Dense(3, activation="relu", name="p2" )(x)
 preds = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1),
name="preds" )([p1, p2])
 model = M.Model(z, preds, name="CNN")
 model.compile(loss=mloss(0.8), optimizer=tf.keras.optimizers.Adam(lr=0.1, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.01, amsgrad=False), metrics=[score])
 return model
import skimage.io
import numpy as np
import pandas as pd
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.utils.data import DataLoader, Dataset
from efficientnet_pytorch import model as enet
import matplotlib.pyplot as plt
from tqdm.notebook import tqdm
from scipy import stats<load_from_csv>
kf = KFold(n_splits=5)
 cnt = 0
EPOCHS = 800
for tr_idx, val_idx in kf.split(X):
cnt += 1
print(f"FOLD {cnt}")
 model = make_model(nh)
 model.fit(X[tr_idx], y[tr_idx], batch_size=128, epochs=EPOCHS,
validation_data=(X[val_idx], y[val_idx]), verbose=0)
 print("train", model.evaluate(X[tr_idx], y[tr_idx], verbose=0, batch_size=128))
print("val", model.evaluate(X[val_idx], y[val_idx], verbose=0, batch_size=128))
print("predict val...")
 pred[val_idx] = model.predict(X[val_idx], batch_size=128, verbose=0)
 print("predict test...")
 pe += model.predict(test_, batch_size=128, verbose=0)/ 5
data_dir = '.. /input/prostate-cancer-grade-assessment'
df_train = pd.read_csv(os.path.join(data_dir, 'train.csv'))
df_test = pd.read_csv(os.path.join(data_dir, 'test.csv'))
df_sub = pd.read_csv(os.path.join(data_dir, 'sample_submission.csv'))
model_dir = ''
image_folder = os.path.join(data_dir, 'test_images')
 is_test = os.path.exists(image_folder)
 image_folder = image_folder if is_test else os.path.join(data_dir, 'train_images')
 df = df_test if is_test else df_train.loc[:100]
tile_size = 256
image_size = 256
n_tiles = 36
batch_size = 8
num_workers = 4
device = torch.device('cuda')
 print(image_folder )<define_search_model>
sigma_opt = mean_absolute_error(y, pred[:, 1])
 unc = pred[:,2] - pred[:, 0]
sigma_mean = np.mean(unc)
 print(sigma_opt, sigma_mean )
class enetv2(nn.Module):
def __init__(self, backbone, out_dim):
super(enetv2, self ).__init__()
self.enet = enet.EfficientNet.from_name(backbone)
 self.myfc = nn.Linear(self.enet._fc.in_features, out_dim)
 self.enet._fc = nn.Identity()
def extract(self, x):
return self.enet(x)
 def forward(self, x):
x = self.extract(x)
 x = self.myfc(x)
 return x
def load_models(model_files):
models = []
for model_f in model_files:
model_f = os.path.join(model_dir, model_f)
 backbone = 'efficientnet-b0'
model = enetv2(backbone, out_dim=5)
 model.load_state_dict(torch.load(model_f, map_location=lambda storage, loc: storage), strict=False)
 model.eval()
model.to(device)
 models.append(model)
 print(f'{model_f} loaded!')
 return models
model_files = [
'.. /input/panda-public-models/cls_effnet_b0_Rand36r36tiles256_big_bce_lr0.3_augx2_30epo_model_fold0.pth',
'.. /input/efficientnet-pytorch/efficientnet-b7-dcc49843.pth'
]
models = load_models(model_files )<categorify>
sample['FVC1'] = 0.996*pe[:, 1]
sample['Confidence1'] = pe[:, 2] - pe[:, 0]
subm = sample[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()
def get_tiles(img, mode=0):
result = []
h, w, c = img.shape
pad_h =(tile_size - h % tile_size)% tile_size +(( tile_size * mode)// 2)
 pad_w =(tile_size - w % tile_size)% tile_size +(( tile_size * mode)// 2)
 img2 = np.pad(img,[[pad_h // 2, pad_h - pad_h // 2], [pad_w // 2,pad_w - pad_w//2], [0,0]], constant_values=255)
 img3 = img2.reshape(
 img2.shape[0] // tile_size,
tile_size,
img2.shape[1] // tile_size,
tile_size,
3 
)
 img3 = img3.transpose(0,2,1,3,4 ).reshape(-1, tile_size, tile_size,3)
 n_tiles_with_info =(img3.reshape(img3.shape[0],-1 ).sum(1)< tile_size ** 2 * 3 * 255 ).sum()
if len(img)< n_tiles:
img3 = np.pad(img3,[[0,N-len(img3)],[0,0],[0,0],[0,0]], constant_values=255)
 idxs = np.argsort(img3.reshape(img3.shape[0],-1 ).sum(-1)) [:n_tiles]
img3 = img3[idxs]
for i in range(len(img3)) :
result.append({'img':img3[i], 'idx':i})
 return result, n_tiles_with_info >= n_tiles
class PANDADataset(Dataset):
def __init__(self,
df,
image_size,
n_tiles=n_tiles,
tile_mode=0,
rand=False,
sub_imgs=False 
):
self.df = df.reset_index(drop=True)
 self.image_size = image_size
self.n_tiles = n_tiles
self.tile_mode = tile_mode
self.rand = rand
self.sub_imgs = sub_imgs
def __len__(self):
return self.df.shape[0]
def __getitem__(self, index):
row = self.df.iloc[index]
img_id = row.image_id
tiff_file = os.path.join(image_folder, f'{img_id}.tiff')
 image = skimage.io.MultiImage(tiff_file)[1]
tiles, OK = get_tiles(image, self.tile_mode)
 if self.rand:
idxes = np.random.choice(list(range(self.n_tiles)) , self.n_tiles, replace=False)
 else:
idxes = list(range(self.n_tiles))
idxes = np.asarray(idxes)+ self.n_tiles if self.sub_imgs else idxes
n_row_tiles = int(np.sqrt(self.n_tiles))
images = np.zeros(( image_size * n_row_tiles, image_size * n_row_tiles, 3))
for h in range(n_row_tiles):
for w in range(n_row_tiles):
i = h * n_row_tiles + w
if len(tiles)> idxes[i]:
this_img = tiles[idxes[i]]['img']
else:
this_img = np.ones(( self.image_size, self.image_size, 3)).astype(np.uint8)* 255
this_img = 255 - this_img
h1 = h * image_size
w1 = w * image_size
images[h1:h1+image_size, w1:w1+image_size] = this_img
images = images.astype(np.float32)
 images /= 255
images = images.transpose(2, 0, 1)
 return torch.tensor(images)
<load_pretrained>
subm.loc[~subm.FVC1.isnull() ,'FVC'] = subm.loc[~subm.FVC1.isnull() ,'FVC1']
if sigma_mean<70:
subm['Confidence'] = sigma_opt
else:
subm.loc[~subm.FVC1.isnull() ,'Confidence'] = subm.loc[~subm.FVC1.isnull() ,'Confidence1']
dataset = PANDADataset(df, image_size, n_tiles, 0)
 loader = DataLoader(dataset, batch_size=batch_size, num_workers=num_workers, shuffle=False)
 dataset2 = PANDADataset(df, image_size, n_tiles, 2)
 loader2 = DataLoader(dataset2, batch_size=batch_size, num_workers=num_workers, shuffle=False )<save_to_csv>
otest = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 for i in range(len(otest)) :
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1
<set_options><EOS> 
subm[["Patient_Week","FVC","Confidence"]].to_csv("submission.csv", index=False )