Kaggle competition: Prostate cANcer graDe Assessment (PANDA) Challenge
<SOS> metric: QuadraticWeightedKappa Kaggle data source: prostate-cancer-grade-assessment-(panda)-challenge<feature_engineering>
sys.path = [
'.. /input/efficientnet-pytorch/EfficientNet-PyTorch/EfficientNet-PyTorch-master',
] + sys.path
xgbtestpreds['id'] = xgbtestpreds["air_store_id"]+'_'+xgbtestpreds["visit_date"].map(str)
 xgbtestpreds['id'] = xgbtestpreds.id.str[:-9]
xgbtestpreds['visitors'] = np.expm1(xgby )<save_to_csv>
sys.path = [
'.. /input/ttach-kaggle/ttach/',
] + sys.path
xgbtestpreds[['id','visitors']].to_csv('xgbtimeseries.csv',index=False )<load_from_csv>
import skimage.io
import numpy as np
import pandas as pd
import torchvision
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.utils.data import DataLoader, Dataset
from efficientnet_pytorch import model as enet
import matplotlib.pyplot as plt
from tqdm import tqdm_notebook as tqdm
data = {
'tra': pd.read_csv('.. /input/air_visit_data.csv'),
'as': pd.read_csv('.. /input/air_store_info.csv'),
'hs': pd.read_csv('.. /input/hpg_store_info.csv'),
'ar': pd.read_csv('.. /input/air_reserve.csv'),
'hr': pd.read_csv('.. /input/hpg_reserve.csv'),
'id': pd.read_csv('.. /input/store_id_relation.csv'),
'tes': pd.read_csv('.. /input/sample_submission.csv'),
'hol': pd.read_csv('.. /input/date_info.csv' ).rename(columns={'calendar_date':'visit_date'})
 }<merge>
data_dir = '.. /input/prostate-cancer-grade-assessment'
df_train = pd.read_csv(os.path.join(data_dir, 'train.csv'))
df_test = pd.read_csv(os.path.join(data_dir, 'test.csv'))
df_sub = pd.read_csv(os.path.join(data_dir, 'sample_submission.csv'))
model_dir = '.. /input/panda-public-models'
image_folder = os.path.join(data_dir, 'test_images')
 is_test = os.path.exists(image_folder)
 image_folder = image_folder if is_test else os.path.join(data_dir, 'train_images')
 df = df_test if is_test else df_train.loc[:16]
tile_size = 256
image_size = 256
n_tiles = 36
batch_size = 4
num_workers = 4
device = torch.device('cuda')
 print(image_folder )
data['hr'] = pd.merge(data['hr'], data['id'], how='inner', on=['hpg_store_id'] )<count_missing_values>
class enetv3(nn.Module):
def __init__(self, modelname, out_dim=5, freeze_bn=True):
super(enetv3, self ).__init__()
self.basemodel = EfficientNet.from_name(modelname)
 self.myfc = nn.Linear(self.basemodel._fc.in_features, out_dim)
 self.basemodel._fc = nn.Identity()
self.basemodel._avg_pooling = nn.AdaptiveAvgPool2d(1)
 def extract(self, x):
return self.basemodel(x)
 def forward(self, x):
x = self.basemodel(x)
 if not "eff" in modelname:
x = self.pool(x ).squeeze(2 ).squeeze(2)
 x = self.myfc(x)
 else:
x = self.myfc(x)
 return x
def load_models(model_files):
models = []
for model_f in model_files:
model_f = os.path.join(".. /input/latesubspanda", model_f)
 if "res" in model_f:
model = enetv2(out_dim=5)
 elif "eff" in model_f:
model = enetv3(modelname, out_dim=5)
 model.load_state_dict(torch.load(model_f))
model.eval()
model.to(device)
 models.append(model)
 print(f'{model_f} loaded!')
 return models
modelname="efficientnet-b0"
model_files2 = [
"efficientnet-b0famlabelsmodelsub_avgpool_tile36_imsize256_mixup_final_epoch20_fold0.pth",
"efficientnet-b0famlabelsmodelsub_avgpool_tile36_imsize256_mixup_final_epoch20_fold1.pth",
"efficientnet-b0famlabelsmodelsub_avgpool_tile36_imsize256_mixup_final_epoch20_fold2.pth",
"efficientnet-b0famlabelsmodelsub_avgpool_tile36_imsize256_mixup_final_epoch20_fold3.pth",
"efficientnet-b0famlabelsmodelsub_avgpool_tile36_imsize256_mixup_final_epoch20_fold4.pth"
]
models2 = load_models(model_files2 )
data['hr'].isnull().values.any()<data_type_conversions>
def get_tiles(img, mode=0):
result = []
h, w, c = img.shape
pad_h =(tile_size - h % tile_size)% tile_size +(( tile_size * mode)// 2)
 pad_w =(tile_size - w % tile_size)% tile_size +(( tile_size * mode)// 2)
 img2 = np.pad(img,[[pad_h // 2, pad_h - pad_h // 2], [pad_w // 2,pad_w - pad_w//2], [0,0]], constant_values=255)
 img3 = img2.reshape(
 img2.shape[0] // tile_size,
tile_size,
img2.shape[1] // tile_size,
tile_size,
3 
)
 img3 = img3.transpose(0,2,1,3,4 ).reshape(-1, tile_size, tile_size,3)
 n_tiles_with_info =(img3.reshape(img3.shape[0],-1 ).sum(1)< tile_size ** 2 * 3 * 255 ).sum()
if len(img)< n_tiles:
img3 = np.pad(img3,[[0,N-len(img3)],[0,0],[0,0],[0,0]], constant_values=255)
 idxs = np.argsort(img3.reshape(img3.shape[0],-1 ).sum(-1)) [:n_tiles]
img3 = img3[idxs]
for i in range(len(img3)) :
result.append({'img':img3[i], 'idx':i})
 return result, n_tiles_with_info >= n_tiles
class PANDADataset(Dataset):
def __init__(self,
df,
image_size,
n_tiles=n_tiles,
tile_mode=0,
rand=False,
sub_imgs=False 
):
self.df = df.reset_index(drop=True)
 self.image_size = image_size
self.n_tiles = n_tiles
self.tile_mode = tile_mode
self.rand = rand
self.sub_imgs = sub_imgs
def __len__(self):
return self.df.shape[0]
def __getitem__(self, index):
row = self.df.iloc[index]
img_id = row.image_id
tiff_file = os.path.join(image_folder, f'{img_id}.tiff')
 image = skimage.io.MultiImage(tiff_file)[1][:,:,::-1]
tiles, OK = get_tiles(image, self.tile_mode)
 if self.rand:
idxes = np.random.choice(list(range(self.n_tiles)) , self.n_tiles, replace=False)
 else:
idxes = list(range(self.n_tiles))
idxes = np.asarray(idxes)+ self.n_tiles if self.sub_imgs else idxes
n_row_tiles = int(np.sqrt(self.n_tiles))
images = np.zeros(( image_size * n_row_tiles, image_size * n_row_tiles, 3))
for h in range(n_row_tiles):
for w in range(n_row_tiles):
i = h * n_row_tiles + w
if len(tiles)> idxes[i]:
this_img = tiles[idxes[i]]['img']
else:
this_img = np.ones(( self.image_size, self.image_size, 3)).astype(np.uint8)* 255
this_img = 255 - this_img
h1 = h * image_size
w1 = w * image_size
images[h1:h1+image_size, w1:w1+image_size] = this_img
images = images.astype(np.float32)
 images /= 255
images = images.transpose(2, 0, 1)
 return torch.tensor(images )
for df in ['ar','hr']:
data[df]['visit_datetime'] = pd.to_datetime(data[df]['visit_datetime'])
 data[df]['visit_datetime'] = data[df]['visit_datetime'].dt.date
data[df]['reserve_datetime'] = pd.to_datetime(data[df]['reserve_datetime'])
 data[df]['reserve_datetime'] = data[df]['reserve_datetime'].dt.date
data[df]['reserve_datetime_diff'] = data[df].apply(lambda r:(r['visit_datetime'] - r['reserve_datetime'] ).days, axis=1 )<merge>
dataset = PANDADataset(df, image_size, n_tiles, 0)
 loader = DataLoader(dataset, batch_size=batch_size, num_workers=num_workers, shuffle=False)
 dataset2 = PANDADataset(df, image_size, n_tiles, 2)
 loader2 = DataLoader(dataset2, batch_size=batch_size, num_workers=num_workers, shuffle=False )
for df in ['ar','hr']:
tmp1 = data[df].groupby(['air_store_id','visit_datetime'], as_index=False)[['reserve_datetime_diff', 'reserve_visitors']].sum().rename(columns={'visit_datetime':'visit_date', 'reserve_visitors':'rv1'})
 tmp2 = data[df].groupby(['air_store_id','visit_datetime'], as_index=False)[['reserve_datetime_diff', 'reserve_visitors']].mean().rename(columns={'visit_datetime':'visit_date', 'reserve_visitors':'rv2'})
 data[df] = pd.merge(tmp1, tmp2, how='inner', on=['air_store_id','visit_date'] )<feature_engineering>
if len(dataset)> 50:
transforms = tta.Compose(
 [
tta.HorizontalFlip() ,
tta.VerticalFlip() ,
tta.Rotate90(angles=[0, 90, 180]),
] 
)
 else:
transforms = tta.Compose(
 [
] 
)
 tta_models2 = []
for model in models2:
tta_models2.append(tta.ClassificationTTAWrapper(model, transforms))
data['tra']['visit_date'] = pd.to_datetime(data['tra']['visit_date'])
 data['tra']['dow'] = data['tra']['visit_date'].dt.dayofweek
data['tra']['year'] = data['tra']['visit_date'].dt.year
data['tra']['month'] = data['tra']['visit_date'].dt.month
data['tra']['visit_date'] = data['tra']['visit_date'].dt.date<feature_engineering>
LOGITS = []
LOGITS2 = []
LOGITS12 = []
LOGITS22 = []
with torch.no_grad() :
for data in tqdm(loader):
data = data.to(device)
 for i, tta_model in enumerate(tta_models2):
if i == 0:
logits = tta_model(data ).sigmoid()
else:
logits = logits + tta_model(data ).sigmoid()
LOGITS2.append(logits/5)
 for data in tqdm(loader2):
data = data.to(device)
 for i, tta_model in enumerate(tta_models2):
if i == 0:
logits = tta_model(data ).sigmoid()
else:
logits = logits + tta_model(data ).sigmoid()
LOGITS22.append(logits/5)
 LOGITS =(torch.cat(LOGITS2 ).cpu() + torch.cat(LOGITS22 ).cpu())/ 2
data['tes']['visit_date'] = data['tes']['id'].map(lambda x: str(x ).split('_')[2])
 data['tes']['air_store_id'] = data['tes']['id'].map(lambda x: '_'.join(x.split('_')[:2]))
data['tes']['visit_date'] = pd.to_datetime(data['tes']['visit_date'])
 data['tes']['dow'] = data['tes']['visit_date'].dt.dayofweek
data['tes']['year'] = data['tes']['visit_date'].dt.year
data['tes']['month'] = data['tes']['visit_date'].dt.month
data['tes']['visit_date'] = data['tes']['visit_date'].dt.date<concatenate>
PREDS = LOGITS.sum(1 ).numpy()
df['isup_grade'] = PREDS
df
stores = pd.concat([pd.DataFrame({'air_store_id': unique_stores, 'dow': [i]*len(unique_stores)})for i in range(7)], axis=0, ignore_index=True ).reset_index(drop=True )<define_variables>
df['isup_grade'] = df['isup_grade'].apply(lambda x: int(np.round(x)) )
<merge><EOS> 
df[['image_id', 'isup_grade']].to_csv('submission.csv', index=False)
 print(df.head())
 print()
print(df.isup_grade.value_counts() )