Kaggle competition: Prostate cANcer graDe Assessment (PANDA) Challenge
<SOS> metric: QuadraticWeightedKappa Kaggle data source: prostate-cancer-grade-assessment-(panda)-challenge<feature_engineering>
sys.path = [
'.. /input/efficientnet-pytorch/EfficientNet-PyTorch/EfficientNet-PyTorch-master',
] + sys.path
weather['NightLength_MPrec']= 24.0 - weather.DayLength_MPrec<feature_engineering>
import numpy as np
import pandas as pd
import torch.nn.functional as F
import os
import torch
import torch.nn as nn
from torch.utils.data import Dataset,DataLoader
from torchvision import transforms,models
from tqdm import tqdm_notebook as tqdm
import math
import torch.utils.model_zoo as model_zoo
import cv2
from sklearn.metrics import cohen_kappa_score
import openslide
import skimage.io
import random
from sklearn.metrics import cohen_kappa_score
import albumentations
from PIL import Image
weather['NightLength_NearH']= 24.0 - weather.DayLength_NearH<not_enough_vertices>
class config:
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 IMG_WIDTH = 512
IMG_HEIGHT = 512
TEST_BATCH_SIZE = 16
CLASSES = 6
DEBUG = False
hours_RiseSet_func = lambda x: x.minute/60.0 + float(x.hour )<feature_engineering>
BASE_PATH = '.. /input/prostate-cancer-grade-assessment'
data_dir = f'{BASE_PATH}/test_images'
test = pd.read_csv(f'{BASE_PATH}/test.csv')
 submission = pd.read_csv(f'{BASE_PATH}/sample_submission.csv')
 if config.DEBUG:
data_dir = f'{BASE_PATH}/train_images'
test = pd.read_csv(f'{BASE_PATH}/train.csv' ).head(200 )
weather['Sunrise_hours'] = weather.Sunrise.apply(hours_RiseSet_func )<feature_engineering>
def seed_torch(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_torch(seed=42 )
weather['Sunset_hours'] = weather.Sunset.apply(hours_RiseSet_func )<define_variables>
from efficientnet_pytorch import EfficientNet
mean_func = lambda x: x.mean()
blend_cols = ['Tmax', 'Tmin', 'Depart' ,'Heat', 'Cool', 'PrecipTotal']<define_variables>
class EfficientNetB3(nn.Module):
def __init__(self, pretrained):
super(EfficientNetB3, self ).__init__()
if pretrained == True:
self.model = EfficientNet.from_name('efficientnet-b3')
 self.model.load_state_dict(torch.load('.. /input/efficientnet-pytorch/efficientnet-b3-c8376fa2.pth'))
else:
self.model = EfficientNet.from_pretrained(None)
 in_features = self.model._fc.in_features
self.l0 = nn.Linear(in_features, config.CLASSES)
 def forward(self, x):
bs, _, _, _ = x.shape
x = self.model.extract_features(x)
 x = F.adaptive_avg_pool2d(x, 1 ).reshape(bs, -1)
 l0 = self.l0(x)
 return l0
class EfficientNetB0(nn.Module):
def __init__(self, pretrained):
super(EfficientNetB0, self ).__init__()
if pretrained == True:
self.model = EfficientNet.from_name('efficientnet-b0')
 self.model.load_state_dict(torch.load('.. /input/efficientnet-pytorch/efficientnet-b0-08094119.pth'))
else:
self.model = EfficientNet.from_pretrained(None)
 in_features = self.model._fc.in_features
self.l0 = nn.Linear(in_features, config.CLASSES)
 def forward(self, x):
bs, _, _, _ = x.shape
x = self.model.extract_features(x)
 x = F.adaptive_avg_pool2d(x, 1 ).reshape(bs, -1)
 l0 = self.l0(x)
 return l0
blended_cols= ['blended_' + col for col in blend_cols]<filter>
def get_tiles(img, mode=0):
result = []
h, w, c = img.shape
pad_h =(tile_size - h % tile_size)% tile_size +(( tile_size * mode)// 2)
 pad_w =(tile_size - w % tile_size)% tile_size +(( tile_size * mode)// 2)
 img2 = np.pad(img,[[pad_h // 2, pad_h - pad_h // 2], [pad_w // 2,pad_w - pad_w//2], [0,0]], constant_values=255)
 img3 = img2.reshape(
 img2.shape[0] // tile_size,
tile_size,
img2.shape[1] // tile_size,
tile_size,
3 
)
 img3 = img3.transpose(0,2,1,3,4 ).reshape(-1, tile_size, tile_size,3)
 n_tiles_with_info =(img3.reshape(img3.shape[0],-1 ).sum(1)< tile_size ** 2 * 3 * 255 ).sum()
if len(img3)< n_tiles:
img3 = np.pad(img3,[[0,n_tiles-len(img3)],[0,0],[0,0],[0,0]], constant_values=255)
 idxs = np.argsort(img3.reshape(img3.shape[0],-1 ).sum(-1)) [:n_tiles]
img3 = img3[idxs]
for i in range(len(img3)) :
result.append({'img':img3[i], 'idx':i})
 return result, n_tiles_with_info >= n_tiles
station_1 = weather[blend_cols][weather.Station == '1']
station_2 = weather[blend_cols][weather.Station == '2']<create_dataframe>
def write_image(path, img):
img = cv2.convertScaleAbs(img, alpha=(255.0))
cv2.imwrite(path, cv2.cvtColor(img, cv2.COLOR_RGB2BGR))
station_blend = pd.DataFrame(( station_1.values + station_2.values)/2, columns= blended_cols )<filter>
tile_size = 256
image_size = 256
n_tiles = 16
batch_size = 2
extract_2 = weather[weather.Station == '2'].reset_index(drop = True)
 extract_2.head()<filter>
class PANDADataset(Dataset):
def __init__(self,
df,
image_size,
n_tiles=n_tiles,
tile_mode=0,
rand=False, 
):
self.df = df.reset_index(drop=True)
 self.image_size = image_size
self.n_tiles = n_tiles
self.tile_mode = tile_mode
self.rand = rand
self.aug = albumentations.Compose([
albumentations.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225], always_apply=True)
 ])
 def __len__(self):
return self.df.shape[0]
def __getitem__(self, index):
row = self.df.iloc[index]
img_id = row.image_id
tiff_file = os.path.join(data_dir, f'{img_id}.tiff')
 image = skimage.io.MultiImage(tiff_file)[1]
tiles, OK = get_tiles(image, self.tile_mode)
 if self.rand:
idxes = np.random.choice(list(range(self.n_tiles)) , self.n_tiles, replace=False)
 else:
idxes = list(range(self.n_tiles))
n_row_tiles = int(np.sqrt(self.n_tiles))
images = np.zeros(( image_size * n_row_tiles, image_size * n_row_tiles, 3))
for h in range(n_row_tiles):
for w in range(n_row_tiles):
i = h * n_row_tiles + w
if len(tiles)> idxes[i]:
this_img = tiles[idxes[i]]['img']
else:
this_img = np.ones(( self.image_size, self.image_size, 3)).astype(np.uint8)* 255
this_img = 255 - this_img
h1 = h * image_size
w1 = w * image_size
images[h1:h1+image_size, w1:w1+image_size] = this_img
images = images.astype(np.float32)
 images /= 255
images = images.transpose(2, 0, 1)
 img = images
img = np.transpose(img,(1, 2, 0))
img = 1 - img
img = cv2.resize(img,(768, 768))
write_image(f'{img_id}.png', img)
 img = skimage.io.MultiImage(f'{img_id}.png')[-1]
img = Image.fromarray(img ).convert("RGB")
 img = self.aug(image=np.array(img)) ["image"]
img = np.transpose(img,(2, 0, 1)).astype(np.float32)
 return { 'image': torch.tensor(img, dtype=torch.float)}
extract_1 = weather[weather.Station == '1'].reset_index(drop = True)
 extract_1.head()<merge>
ENSEMBLES = [
{
'model_name': 'efficientnet-b0',
'model_weight': 'efficientnetB0_0.pth',
'ensemble_weight': 1
}
]
joined_1 = extract_1.join(station_blend)
 joined_2 = extract_2.join(station_blend )<concatenate>
device = config.device
models = []
for ensemble in ENSEMBLES:
model = EfficientNetB3(pretrained=True)
 model.load_state_dict(torch.load(ensemble['model_weight'], map_location=device))
model.to(device)
 models.append(model )
weather_blend = pd.concat([joined_1, joined_2] )<feature_engineering>
model.eval()
predictions = []
if check_for_images_dir() :
test_dataset = PANDADataset(
 df=test,
image_size=image_size,
n_tiles=n_tiles,
tile_mode=0 
)
 test_data_loader = torch.utils.data.DataLoader(
 test_dataset,
batch_size=config.TEST_BATCH_SIZE,
shuffle=False, 
)
 for model in models:
preds = []
for idx, d in tqdm(enumerate(test_data_loader), total=len(test_data_loader)) :
inputs = d["image"]
inputs = inputs.to(device)
 with torch.no_grad() :
outputs = model(inputs)
 preds.append(outputs.to('cpu' ).numpy())
 predictions.append(np.concatenate(preds))
predictions = np.mean(predictions, axis=0)
 predictions = predictions.argmax(1)
month_func = lambda x: x.month
day_func= lambda x: x.day
day_of_year_func = lambda x: x.dayofyear
week_of_year_func = lambda x: x.week
train['month'] = train.Date.apply(month_func)
 train['day'] = train.Date.apply(day_func)
 train['day_of_year'] = train.Date.apply(day_of_year_func)
 train['week'] = train.Date.apply(week_of_year_func)
 test['month'] = test.Date.apply(month_func)
 test['day'] = test.Date.apply(day_func)
 test['day_of_year'] = test.Date.apply(day_of_year_func)
 test['week'] = test.Date.apply(week_of_year_func )<drop_column>
if config.DEBUG:
def quadratic_weighted_kappa(y_hat, y):
return cohen_kappa_score(y_hat, y, weights='quadratic')
 count = 0
for index, val in enumerate(test.isup_grade.values):
if predictions[index] == val:
count += 1
print(f"Accuracy Train is {(count / test.shape[0])* 100}")
 print(f'Kappa Train is {quadratic_weighted_kappa(predictions, test.isup_grade.values)}')
 else:
if len(predictions)> 0:
submission.isup_grade = predictions
submission.isup_grade = submission['isup_grade'].astype(int)
 submission.to_csv('submission.csv',index=False)
 print(submission.head() )
weather_blend = weather_blend.drop(['Sunrise', 'Sunset'], axis= 1 )<merge>
sys.path = [
'.. /input/efficientnet-pytorch/EfficientNet-PyTorch/EfficientNet-PyTorch-master',
] + sys.path
train = train.merge(weather_blend, on='Date')
 test = test.merge(weather_blend, on='Date' )<define_variables>
import numpy as np
import pandas as pd
import torch.nn.functional as F
import os
import torch
import torch.nn as nn
from torch.utils.data import Dataset,DataLoader
from torchvision import transforms,models
from tqdm import tqdm_notebook as tqdm
import math
import torch.utils.model_zoo as model_zoo
import cv2
from sklearn.metrics import cohen_kappa_score
import openslide
import skimage.io
import random
from sklearn.metrics import cohen_kappa_score
import albumentations
from PIL import Image
cols_to_write = [col for col in train.columns if col != 'Date']<filter>
class config:
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 IMG_WIDTH = 512
IMG_HEIGHT = 512
TEST_BATCH_SIZE = 16
CLASSES = 6
DEBUG = False
train_station_1= train[train.Station == '1']
train_station_2= train[train.Station == '2']
test_station_1= test[test.Station == '1']
test_station_2= test[test.Station == '2']<save_to_csv>
BASE_PATH = '.. /input/prostate-cancer-grade-assessment'
data_dir = f'{BASE_PATH}/test_images'
test = pd.read_csv(f'{BASE_PATH}/test.csv')
 submission = pd.read_csv(f'{BASE_PATH}/sample_submission.csv')
 if config.DEBUG:
data_dir = f'{BASE_PATH}/train_images'
test = pd.read_csv(f'{BASE_PATH}/train.csv' ).head(200 )
train.to_csv('train.csv' )<rename_columns>
def seed_torch(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_torch(seed=42 )
keep_cols = ['Date', u'Tmax', u'Tmin', u'Tavg',u'PrecipTotal']
train_station_2 = train_station_2[keep_cols]
test_station_2 = test_station_2[keep_cols]
prefix_s2 = 'stat_2_'
rename_cols_s2 = [prefix_s2 + col for col in train_station_2.columns]
train_station_2.columns = rename_cols_s2
test_station_2.columns = rename_cols_s2<drop_column>
from efficientnet_pytorch import EfficientNet
drop_cols = ['Heat', 'Cool', 'Depart', 'NightLength_MPrec', 'NightLength_NearH',
'blended_Depart', 'blended_Heat', 'blended_Cool']
train_station_1 = train_station_1.drop(drop_cols, axis= 1)
 test_station_1 = test_station_1.drop(drop_cols, axis= 1 )<rename_columns>
class EfficientNetB3(nn.Module):
def __init__(self, pretrained):
super(EfficientNetB3, self ).__init__()
if pretrained == True:
self.model = EfficientNet.from_name('efficientnet-b3')
 self.model.load_state_dict(torch.load('.. /input/efficientnet-pytorch/efficientnet-b3-c8376fa2.pth'))
else:
self.model = EfficientNet.from_pretrained(None)
 in_features = self.model._fc.in_features
self.l0 = nn.Linear(in_features, config.CLASSES)
 def forward(self, x):
bs, _, _, _ = x.shape
x = self.model.extract_features(x)
 x = F.adaptive_avg_pool2d(x, 1 ).reshape(bs, -1)
 l0 = self.l0(x)
 return l0
class EfficientNetB0(nn.Module):
def __init__(self, pretrained):
super(EfficientNetB0, self ).__init__()
if pretrained == True:
self.model = EfficientNet.from_name('efficientnet-b0')
 self.model.load_state_dict(torch.load('.. /input/efficientnet-pytorch/efficientnet-b0-08094119.pth'))
else:
self.model = EfficientNet.from_pretrained(None)
 in_features = self.model._fc.in_features
self.l0 = nn.Linear(in_features, config.CLASSES)
 def forward(self, x):
bs, _, _, _ = x.shape
x = self.model.extract_features(x)
 x = F.adaptive_avg_pool2d(x, 1 ).reshape(bs, -1)
 l0 = self.l0(x)
 return l0
prefix_s1 = 'stat_1_'
rename_cols_s1 = [prefix_s1 + col for col in keep_cols]
cols_to_rename= [col for col in train_station_1.columns if col in keep_cols]
s1_name_map = dict(zip(cols_to_rename, rename_cols_s1))
train_station_1 = train_station_1.rename(columns= s1_name_map)
 test_station_1 = test_station_1.rename(columns= s1_name_map )<concatenate>
def get_tiles(img, mode=0):
result = []
h, w, c = img.shape
pad_h =(tile_size - h % tile_size)% tile_size +(( tile_size * mode)// 2)
 pad_w =(tile_size - w % tile_size)% tile_size +(( tile_size * mode)// 2)
 img2 = np.pad(img,[[pad_h // 2, pad_h - pad_h // 2], [pad_w // 2,pad_w - pad_w//2], [0,0]], constant_values=255)
 img3 = img2.reshape(
 img2.shape[0] // tile_size,
tile_size,
img2.shape[1] // tile_size,
tile_size,
3 
)
 img3 = img3.transpose(0,2,1,3,4 ).reshape(-1, tile_size, tile_size,3)
 n_tiles_with_info =(img3.reshape(img3.shape[0],-1 ).sum(1)< tile_size ** 2 * 3 * 255 ).sum()
if len(img3)< n_tiles:
img3 = np.pad(img3,[[0,n_tiles-len(img3)],[0,0],[0,0],[0,0]], constant_values=255)
 idxs = np.argsort(img3.reshape(img3.shape[0],-1 ).sum(-1)) [:n_tiles]
img3 = img3[idxs]
for i in range(len(img3)) :
result.append({'img':img3[i], 'idx':i})
 return result, n_tiles_with_info >= n_tiles
train_station_1 = train_station_1.reset_index(drop= True)
 train_station_2 = train_station_2.reset_index(drop = True)
 train_merge = pd.concat([train_station_1, train_station_2], axis= 1)
 test_station_1 = test_station_1.reset_index(drop= True)
 test_station_2 = test_station_2.reset_index(drop = True)
 test_merge = pd.concat([test_station_1, test_station_2], axis= 1 )<drop_column>
def write_image(path, img):
img = cv2.convertScaleAbs(img, alpha=(255.0))
cv2.imwrite(path, cv2.cvtColor(img, cv2.COLOR_RGB2BGR))
labels = train_merge.pop('WnvPresent' ).values<drop_column>
tile_size = 256
image_size = 256
n_tiles = 16
batch_size = 2
train_merge = train_merge.drop(['stat_1_Date', 'stat_2_Date'], axis = 1)
 test_merge = test_merge.drop(['stat_1_Date', 'stat_2_Date' ], axis = 1 )<data_type_conversions>
class PANDADataset(Dataset):
def __init__(self,
df,
image_size,
n_tiles=n_tiles,
tile_mode=0,
rand=False, 
):
self.df = df.reset_index(drop=True)
 self.image_size = image_size
self.n_tiles = n_tiles
self.tile_mode = tile_mode
self.rand = rand
self.aug = albumentations.Compose([
albumentations.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225], always_apply=True)
 ])
 def __len__(self):
return self.df.shape[0]
def __getitem__(self, index):
row = self.df.iloc[index]
img_id = row.image_id
tiff_file = os.path.join(data_dir, f'{img_id}.tiff')
 image = skimage.io.MultiImage(tiff_file)[1]
tiles, OK = get_tiles(image, self.tile_mode)
 if self.rand:
idxes = np.random.choice(list(range(self.n_tiles)) , self.n_tiles, replace=False)
 else:
idxes = list(range(self.n_tiles))
n_row_tiles = int(np.sqrt(self.n_tiles))
images = np.zeros(( image_size * n_row_tiles, image_size * n_row_tiles, 3))
for h in range(n_row_tiles):
for w in range(n_row_tiles):
i = h * n_row_tiles + w
if len(tiles)> idxes[i]:
this_img = tiles[idxes[i]]['img']
else:
this_img = np.ones(( self.image_size, self.image_size, 3)).astype(np.uint8)* 255
this_img = 255 - this_img
h1 = h * image_size
w1 = w * image_size
images[h1:h1+image_size, w1:w1+image_size] = this_img
images = images.astype(np.float32)
 images /= 255
images = images.transpose(2, 0, 1)
 img = images
img = np.transpose(img,(1, 2, 0))
img = 1 - img
img = cv2.resize(img,(768, 768))
write_image(f'{img_id}.png', img)
 img = skimage.io.MultiImage(f'{img_id}.png')[-1]
img = Image.fromarray(img ).convert("RGB")
 img = self.aug(image=np.array(img)) ["image"]
img = np.transpose(img,(2, 0, 1)).astype(np.float32)
 return { 'image': torch.tensor(img, dtype=torch.float)}
train_merge['Lat_int'] = train_merge.Latitude.astype(int)
 train_merge['Long_int'] = train_merge.Longitude.astype(int)
 test_merge['Lat_int'] = test_merge.Latitude.astype(int)
 test_merge['Long_int'] = test_merge.Longitude.astype(int )<categorify>
ENSEMBLES = [
{
'model_name': 'efficientnet-b0',
'model_weight': 'efficientnetB0_0.pth',
'ensemble_weight': 1
}
]
train_merge = pd.get_dummies(train_merge, columns= ['Species'])
 train_merge = pd.get_dummies(train_merge, columns= ['Block'])
 train_merge = pd.get_dummies(train_merge, columns= ['Street'])
 train_merge = pd.get_dummies(train_merge, columns= ['Trap'])
 test_merge = pd.get_dummies(test_merge, columns= ['Species'])
 test_merge = pd.get_dummies(test_merge, columns= ['Block'])
 test_merge = pd.get_dummies(test_merge, columns= ['Street'])
 test_merge = pd.get_dummies(test_merge, columns= ['Trap'] )<drop_column>
device = config.device
models = []
for ensemble in ENSEMBLES:
model = EfficientNetB3(pretrained=True)
 model.load_state_dict(torch.load(ensemble['model_weight'], map_location=device))
model.to(device)
 models.append(model )
train_merge= train_merge.drop('Station', axis= 1)
 test_merge= test_merge.drop('Station', axis= 1 )<drop_column>
model.eval()
predictions = []
if check_for_images_dir() :
test_dataset = PANDADataset(
 df=test,
image_size=image_size,
n_tiles=n_tiles,
tile_mode=0 
)
 test_data_loader = torch.utils.data.DataLoader(
 test_dataset,
batch_size=config.TEST_BATCH_SIZE,
shuffle=False, 
)
 for model in models:
preds = []
for idx, d in tqdm(enumerate(test_data_loader), total=len(test_data_loader)) :
inputs = d["image"]
inputs = inputs.to(device)
 with torch.no_grad() :
outputs = model(inputs)
 preds.append(outputs.to('cpu' ).numpy())
 predictions.append(np.concatenate(preds))
predictions = np.mean(predictions, axis=0)
 predictions = predictions.argmax(1)
<define_variables><EOS> 
if config.DEBUG:
def quadratic_weighted_kappa(y_hat, y):
return cohen_kappa_score(y_hat, y, weights='quadratic')
 count = 0
for index, val in enumerate(test.isup_grade.values):
if predictions[index] == val:
count += 1
print(f"Accuracy Train is {(count / test.shape[0])* 100}")
 print(f'Kappa Train is {quadratic_weighted_kappa(predictions, test.isup_grade.values)}')
 else:
if len(predictions)> 0:
submission.isup_grade = predictions
submission.isup_grade = submission['isup_grade'].astype(int)
 submission.to_csv('submission.csv',index=False)
 print(submission.head() )