Kaggle competition: APTOS 2019 Blindness Detection
test_csv = pd.read_csv('.. /input/aptos2019-blindness-detection/sample_submission.csv')
 id_code = test_csv['id_code']
prediction = np.empty(len(id_code), dtype='uint8')
 for i in range(len(id_code)) :
img = cv2.imread('.. /input/aptos2019-blindness-detection/test_images/{}.png'.format(id_code[i]))
img300 = load_ben_color(img, 300 ).astype('float32')/ 255.
 X3 = np.array([img300])
 pred1 = model_b0_256.predict(X3)
 pred2 = model_b1_240.predict(X3)
 pred3 = model_b3_300.predict(X3)
 pred1 = np.mean(pred1)
 pred2 = np.mean(pred2)
 pred3 = np.mean(pred3)
 pred =(pred1 + pred2 + pred3)/ 3
pred = np.rint(pred ).astype('uint8')
 print(pred)
 prediction[i] = pred<save_to_csv>
sys.path.append('.. /input/efficientnet/efficientnet-pytorch/EfficientNet-PyTorch/')
 test_batch_size = 16
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 weights_path = '.. /input/efficientnet-pytorch/efficientnet-b0-08094119.pth'
class OptimizedRounder() :
def __init__(self):
self.coef_ = 0
def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X=X, y=y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 def predict(self, X, coef=[0.5, 1.5, 2.5, 3.5]):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
return X_p
def coefficients(self):
return self.coef_['x']
def predict(X, coef=[0.5, 1.5, 2.5, 3.5]):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
return X_p
def crop_image1(img,tol=7):
mask = img>tol
return img[np.ix_(mask.any(1),mask.any(0)) ]
def crop_image_from_gray(img,tol=7):
if img.ndim ==2:
mask = img>tol
return img[np.ix_(mask.any(1),mask.any(0)) ]
elif img.ndim==3:
gray_img = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
 mask = gray_img>tol
check_shape = img[:,:,0][np.ix_(mask.any(1),mask.any(0)) ].shape[0]
if(check_shape == 0):
return img
else:
img1=img[:,:,0][np.ix_(mask.any(1),mask.any(0)) ]
img2=img[:,:,1][np.ix_(mask.any(1),mask.any(0)) ]
img3=img[:,:,2][np.ix_(mask.any(1),mask.any(0)) ]
img = np.stack([img1,img2,img3],axis=-1)
 return img
def preprocess_image(image_path, desired_size=256):
img = cv2.imread(image_path)
 img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
 img = crop_image_from_gray(img)
 img = cv2.resize(img,(desired_size,desired_size))
img = cv2.addWeighted(img,4,cv2.GaussianBlur(img,(0,0), desired_size/30),-4 ,128)
 return img
def preprocess_image_old(image_path, desired_size=224):
img = cv2.imread(image_path)
 img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
 img = cv2.resize(img,(desired_size,desired_size))
img = cv2.addWeighted(img,4,cv2.GaussianBlur(img,(0,0), desired_size/40),-4 ,128)
 return img
class MyDataset(Dataset):
def __init__(self,dataframe, root_dir,transform=None,train = False):
self.df =dataframe
self.transform = transform
self.train = train
self.root_dir = root_dir
def __len__(self):
return len(self.df)
 def __getitem__(self, idx):
p = self.df.id_code.values[idx]
if idx<10:
p = os.path.join(self.root_dir,str(p)+'.jpg')
 else:
p = os.path.join(self.root_dir,str(p)+'.jpg')
 if self.train:
image = preprocess_image_old(str(p))
else:
image = preprocess_image(str(p))
if self.transform:
image = self.transform(image)
 return image
test_transform = transforms.Compose([
transforms.ToTensor() ,
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])])
 root_dir = '.. /input/xczdatasets/Original_Images/Testing_Set'
train_df = pd.read_csv('.. /input/labels/test.csv')
 testset = MyDataset(train_df,root_dir,transform=test_transform)
 test_loader = torch.utils.data.DataLoader(testset, batch_size=test_batch_size, shuffle=False)
 image = testset[random.randint(0, len(testset)-1)]
class ClassifierModule(nn.Sequential):
def __init__(self, n_features):
super().__init__(
 nn.BatchNorm1d(n_features),
nn.Dropout(0.5),
nn.Linear(n_features, n_features),
nn.PReLU() ,
nn.BatchNorm1d(n_features),
nn.Dropout(0.2),
nn.Linear(n_features, 1), 
)
 class CustomEfficientNet(nn.Module):
def __init__(self, model_name='efficientnet-b0', weights_path=None):
assert model_name in('efficientnet-b0', 'efficientnet-b1', 'efficientnet-b2', 'efficientnet-b3', 'e fficientnet-b4', 'efficientnet-b5')
 super().__init__()
self.net = EfficientNet.from_name(model_name)
 self.net.load_state_dict(torch.load(weights_path))
n_features = self.net._fc.in_features
self.net._fc = ClassifierModule(n_features)
 def forward(self, x):
return self.net(x)
 model = CustomEfficientNet(model_name='efficientnet-b0', weights_path=weights_path)
 model.load_state_dict(torch.load('.. /input/pretrained-model/efficientnet-b0_fold0_epoch12.pth'))
model.cuda()
model.eval()
valid_preds = np.zeros(( len(testset)))
 avg_val_loss = 0.
 for i, images in enumerate(test_loader):
with torch.no_grad() :
y_preds = model(images.to(device)).detach()
valid_preds[i * test_batch_size:(i+1)* test_batch_size] = y_preds[:, 0].to('cpu' ).numpy()
pred = predict(valid_preds)
 print(np.squeeze(pred))
pred_label = pd.DataFrame({'id_code':pd.read_csv('.. /input/labels/test.csv' ).id_code.values,
'labels':np.squeeze(pred ).astype(int)})
 print(pred_label.head())
 pred_label.to_csv('train.csv', index=False )
test_csv['diagnosis'] = prediction
test_csv.to_csv("submission.csv", index=False)
 unique, counts = np.unique(prediction, return_counts=True)
 tmp = dict(zip(unique, counts))
print(tmp )<feature_engineering>
train_params = {
'n_splits': 5,
'n_epochs': 12,
'lr': 1e-3,
'base_lr': 1e-4,
'max_lr': 3e-3,
'step_factor': 6,
'train_batch_size': 32,
'train_image_size': 512,
'test_image_size': 512,
'test_batch_size': 32,
'accumulation_steps': 10,
}
sys.path.append('.. /input')
 sys.modules['efficientnet_pytorch'] = importlib.import_module('efficientnet-pytorch')
 sys.modules['rectified_adam'] = importlib.import_module('rectified-adam' )<import_modules>
from pathlib import Path
import pandas as pd
from fastai.vision import *
from efficientnet_pytorch import EfficientNet
from rectified_adam.radam import *
from functools import partial
import io<define_variables>
sys.path.append('.. /input/efficientnet-my/' )
data_dir = Path('.. /input/aptos2019-blindness-detection')
 batch_size = 8
im_size =(528,528)
 train = False
ssl = False
c = 1/3
model_dir = Path('.. /input/my-aptos2019-blindness-detection' )<load_from_csv>
Compose, HorizontalFlip, IAAAdditiveGaussianNoise, Normalize, OneOf,
RandomBrightness, RandomContrast, Resize, VerticalFlip, Rotate, ShiftScaleRotate,
RandomBrightnessContrast, OpticalDistortion, GridDistortion, ElasticTransform, Cutout 
)
def preprocess(im):
bbox = pil2tensor(im, dtype=np.uint8 ).nonzero().transpose(1,0)
 im = im.crop(( bbox[2].min().item() ,
bbox[1].min().item() ,
bbox[2].max().item() ,
bbox[1].max().item()))
return im
if train:
df = pd.read_csv(data_dir/'train.csv')
 counts = df['diagnosis'].value_counts(normalize=True)
 w = 1 - tensor([counts[i] for i in range(5)])
 w = w.to(defaults.device)
 L = len(df)
 if ssl: df = df.sample(frac=1 - c)
 src = ImageList.from_df(df, data_dir/'train_images', suffix='.png', after_open=preprocess)
 if ssl:
df = pd.read_csv(model_dir/'test.csv' ).sample(n=int(c * L))
src.add(ImageList.from_df(df, data_dir/'test_images', suffix='.png', after_open=preprocess))
data = src.split_none() \ 
.label_from_df(classes=[0,1,2,3,4])\ 
.transform(
 get_transforms(max_warp=None,
xtra_tfms=[cutout(length=(im_size[0]//8,im_size[0]//4)) ]),
size=im_size,
resize_method=ResizeMethod.SQUISH,
padding_mode='zeros')\ 
.databunch(bs=batch_size)\ 
.normalize(stats=imagenet_stats )<load_pretrained>
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 device
model_name = 'efficientnet-b6'
if train:
model = EfficientNet.from_pretrained(model_name, num_classes=5)
 kappa = KappaScore()
kappa.weights = 'quadratic'
learn = Learner(data, model,
loss_func=partial(F.cross_entropy, weight=w), opt_func=RAdam, metrics=[kappa] ).to_fp16()
learn.layer_groups = split_model_idx(learn.model, idxs=[-1])
 learn.summary()
else:
learn = load_learner(model_dir, model_name + '.pkl')
 learn.path = Path('.')
 if ssl:
with io.open(model_dir/f'{model_name}.pth', 'rb')as f:
learn = learn.load(f )<train_model>
def seed_torch(seed=777):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
SEED = 777
seed_torch(SEED )
if train:
if not ssl:
learn.freeze()
learn.fit(1, lr=1e-3)
 nepochs = 12
learn.unfreeze()
learn.fit_one_cycle(cyc_len=nepochs, max_lr=1e-4, div_factor=10, pct_start=0)
 learn.export(model_name + '.pkl')
 learn.save(model_name)
 else:
learn.model.eval()
mean = tensor(imagenet_stats[0])
 std = tensor(imagenet_stats[1])
 def diagnose(row):
fn = row['id_code']
im = open_image(data_dir/f'test_images/{fn}.png', div=True, after_open=preprocess)
 im = im.apply_tfms(None, size=im_size, resize_method=ResizeMethod.PAD, padding_mode='zeros')
 x = torch.stack([
im.data,
])
 x =(x - mean[None,:,None,None])/ std[None,:,None,None]
x = x.to(defaults.device ).half()
pred = learn.model(x ).mean(dim=0)
 pred = np.argmax(pred)
 return pred.item()
df = pd.read_csv(data_dir/'test.csv')
 df['diagnosis'] = df.apply(diagnose, axis=1)
 df.to_csv('submission.csv', index=False, line_terminator='
' )<load_from_csv>
def quadratic_weighted_kappa(y_hat, y):
return cohen_kappa_score(y_hat, y, weights='quadratic' )
warnings.filterwarnings("ignore")
 train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )<drop_column>
class OptimizedRounder() :
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
ll = quadratic_weighted_kappa(y, X_p)
 return -ll
def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X=X, y=y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 def predict(self, X, coef):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
return X_p
def coefficients(self):
return self.coef_['x']
test_ids = test.id
train.drop(['id'], axis=1, inplace=True)
 test.drop(['id'], axis=1, inplace=True )<define_variables>
class CyclicLR(_LRScheduler):
def __init__(self, optimizer, base_lr, max_lr, step_size, gamma=0.99, mode='triangular', last_epoch=-1):
self.optimizer = optimizer
self.base_lr = base_lr
self.max_lr = max_lr
self.step_size = step_size
self.gamma = gamma
self.mode = mode
assert mode in ['triangular', 'triangular2', 'exp_range']
super().__init__(optimizer, last_epoch)
 def get_lr(self):
new_lr = []
for base_lr in self.base_lrs:
cycle = np.floor(1 + self.last_epoch /(2 * self.step_size))
x = np.abs(float(self.last_epoch)/ self.step_size - 2 * cycle + 1)
 if self.mode == 'triangular':
lr = self.base_lr +(self.max_lr - self.base_lr)* np.maximum(0,(1 - x))
elif self.mode == 'triangular2':
lr = self.base_lr +(self.max_lr - self.base_lr)* np.maximum(0,(1 - x)) / float(2 **(cycle - 1))
elif self.mode == 'exp_range':
lr = self.base_lr +(self.max_lr - self.base_lr)* np.maximum(0,(1 - x)) *(self.gamma **(self.last_epoch))
new_lr.append(lr)
 return new_lr
margin_cols = [col for col in train.columns if 'margin' in col]
shape_cols = [col for col in train.columns if 'shape' in col]
texture_cols = [col for col in train.columns if 'texture' in col]<categorify>
X = train.drop('species', axis=1)
 y = train[["species"]]
le=LabelEncoder().fit(y)
 y=le.transform(y )<split>
old_train = old_train[['image','level']]
old_train.columns = new_train.columns
old_train.diagnosis.value_counts()
new_train['id_code'] = '.. /input/aptos2019-blindness-detection/train_images/' + new_train['id_code'].astype(str)+ '.png'
old_train['id_code'] = '.. /input/retinopathy-train-2015/rescaled_train_896/rescaled_train_896/' + old_train['id_code'].astype(str)+ '.png'
train_df = old_train[:992].copy()
val_df = new_train[:480].copy()
train_df.head
print(train_df.shape)
 print(val_df.shape )
sss = StratifiedShuffleSplit(n_splits = 10, test_size=0.2, random_state=123)
 scores = []
k = 0
for train_index, test_index in sss.split(X, y):
X_train, X_test = X.iloc[train_index], X.iloc[test_index]
y_train, y_test = y[train_index], y[test_index]<choose_model_class>
import cv2
import matplotlib.pyplot as plt
import matplotlib
import os
classifiers = [
KNeighborsClassifier(3),
SVC(kernel="rbf", C=0.025, probability=True),
NuSVC(probability=True),
DecisionTreeClassifier() ,
RandomForestClassifier(n_estimators=1000),
AdaBoostClassifier() ,
GradientBoostingClassifier() ,
GaussianNB() ,
LinearDiscriminantAnalysis() ,
QuadraticDiscriminantAnalysis() ,
LogisticRegression()
]
log_cols=["Classifier", "Accuracy", "Log Loss"]
log = pd.DataFrame(columns=log_cols)
 for clf in classifiers:
clf.fit(X_train, y_train)
 name = clf.__class__.__name__
print("+"*30)
 print(name)
 print('****Results****')
 train_predictions = clf.predict(X_test)
 acc = accuracy_score(y_test, train_predictions)
 print("Accuracy: {:.4%}".format(acc))
train_predictions = clf.predict_proba(X_test)
 ll = log_loss(y_test, train_predictions)
 print("Log Loss: {}".format(ll))
log_entry = pd.DataFrame([[name, acc*100, ll]], columns=log_cols)
 log = log.append(log_entry)
 print("+"*30 )<normalization>
root = '/kaggle/input/aptos2019-blindness-detection/'
train_df = pd.read_csv(os.path.join(root, 'train.csv'))
scaler = StandardScaler().fit(X_train)
 X_train_scale = scaler.transform(X_train )<train_on_grid>
class MyDataset(Dataset):
def __init__(self, dataframe, transform=None,train = True):
self.df = dataframe
self.transform = transform
self.train = train
def __len__(self):
return len(self.df)
 def __getitem__(self, idx):
label = self.df.diagnosis.values[idx]
label = np.expand_dims(label, -1)
 label = torch.tensor(label ).float()
p = self.df.id_code.values[idx]
if self.train:
image = preprocess_image_old(str(p))
else:
image = preprocess_image(str(p))
image = transforms.ToPILImage()(image)
 if self.transform:
image = self.transform(image)
 return image, label
params = {'C':[1000, 2000], 'tol': [0.001, 0.0001]}
log_reg = LogisticRegression(solver='newton-cg', multi_class='multinomial',class_weight='balanced',max_iter=400)
 grid_search_lgr = GridSearchCV(log_reg, params, scoring='neg_log_loss', refit='True', n_jobs=-1, cv=5)
 grid_search_lgr.fit(X_train_scale, y_train)
 print('Best score: {}'.format(grid_search_lgr.best_score_))
print('Best parameters: {}'.format(grid_search_lgr.best_params_))<predict_on_test>
train_transform = transforms.Compose([
transforms.RandomHorizontalFlip() ,
transforms.RandomRotation(( -120, 120)) ,
transforms.ToTensor() ,
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])])
X_test_scale = scaler.transform(test)
 y_pred_prob = grid_search_lgr.predict_proba(X_test_scale)
 print(y_pred_prob.shape)
<prepare_output>
submission = pd.DataFrame(y_pred_prob, columns=list(le.classes_))
submission.insert(0, 'id', test_ids)
 submission.reset_index()
submission.head()<save_to_csv>
PRETRAINED_DIR = Path('.. /input/pytorch-pretrained-models')
 EFFICIENTNET_PRETRAINED_DIR = Path('.. /input/efficientnet-pytorch')
 os.listdir('.. /input/efficientnet-pytorch/')
 PRETRAINED_MAPPING = {
'resnet18': PRETRAINED_DIR / 'resnet18-5c106cde.pth',
'resnet34': PRETRAINED_DIR / 'resnet34-333f7ec4.pth',
'resnet50': PRETRAINED_DIR / 'resnet50-19c8e357.pth',
'resnet101': PRETRAINED_DIR / 'resnet101-5d3b4d8f.pth',
'resnet152': PRETRAINED_DIR / 'resnet152-b121ed2d.pth',
'resnext101_32x4d': PRETRAINED_DIR / 'resnext101_32x4d-29e315fa.pth',
'resnext101_64x4d': PRETRAINED_DIR / 'resnext101_64x4d-e77a0586.pth',
'densenet121': PRETRAINED_DIR / 'densenet121-fbdb23505.pth',
'densenet169': PRETRAINED_DIR / 'densenet169-f470b90a4.pth',
'densenet201': PRETRAINED_DIR / 'densenet201-5750cbb1e.pth',
'densenet161': PRETRAINED_DIR / 'densenet161-347e6b360.pth',
'se_resnet50': PRETRAINED_DIR / 'se_resnet50-ce0d4300.pth',
'se_resnet101': PRETRAINED_DIR / 'se_resnet101-7e38fcc6.pth',
'se_resnet152': PRETRAINED_DIR / 'se_resnet152-d17c99b7.pth',
'se_resnext50_32x4d': PRETRAINED_DIR / 'se_resnext50_32x4d-a260b3a4.pth',
'se_resnext101_32x4d': PRETRAINED_DIR / 'se_resnext101_32x4d-3b2fe3d8.pth',
'senet154': PRETRAINED_DIR / 'senet154-c7b49a05.pth',
'inceptionv3': PRETRAINED_DIR / 'inception_v3_google-1a9a5a14.pth',
'inceptionv4': PRETRAINED_DIR / 'inceptionv4-8e4777a0.pth',
'bninception': PRETRAINED_DIR / 'bn_inception-52deb4733.pth',
'inceptionresnetv2': PRETRAINED_DIR / 'inceptionresnetv2-520b38e4.pth',
'xception': PRETRAINED_DIR / 'xception-43020ad28.pth',
'dpn68': PRETRAINED_DIR / 'dpn68-4af7d88d2.pth',
'dpn98': PRETRAINED_DIR / 'dpn98-722954780.pth',
'dpn131': PRETRAINED_DIR / 'dpn131-7af84be88.pth',
'dpn68b': PRETRAINED_DIR / 'dpn68b_extra-363ab9c19.pth',
'dpn92': PRETRAINED_DIR / 'dpn92_extra-fda993c95.pth',
'dpn107': PRETRAINED_DIR / 'dpn107_extra-b7f9f4cc9.pth',
'polynet': PRETRAINED_DIR / 'polynet-f71d82a5.pth',
'nasnetalarge': PRETRAINED_DIR / 'nasnetalarge-a1897284.pth',
'pnasnet5large': PRETRAINED_DIR / 'pnasnet5large-bf079911.pth',
'efficientnet-b0': EFFICIENTNET_PRETRAINED_DIR / 'efficientnet-b0-08094119.pth',
'efficientnet-b1': EFFICIENTNET_PRETRAINED_DIR / 'efficientnet-b1-dbc7070a.pth',
'efficientnet-b2': EFFICIENTNET_PRETRAINED_DIR / 'efficientnet-b2-27687264.pth',
'efficientnet-b3': EFFICIENTNET_PRETRAINED_DIR / 'efficientnet-b3-c8376fa2.pth',
'efficientnet-b4': EFFICIENTNET_PRETRAINED_DIR / 'efficientnet-b4-e116e8b3.pth',
'efficientnet-b5': EFFICIENTNET_PRETRAINED_DIR / 'efficientnet-b5-586e6cc6.pth',
}
submission.to_csv('submission.csv', index=False )<save_to_csv>
submission.to_csv('submission.csv', index=False )<import_modules>
def basic_conv(in_channels, out_channels, kernel_size, stride=1, padding=0, dilation=0, bn_momentum=0.1):
basicconv = nn.Sequential(
 nn.Conv2d(in_channels, out_channels, kernel_size, stride, padding, dilation, bias=False),
nn.BatchNorm2d(out_channels, momentum=bn_momentum),
nn.ReLU() 
)
 return basicconv
class Basic_cell(nn.Module):
def __init__(self,in_channels=1280):
super().__init__()
self.conv1 = nn.Conv2d(in_channels,64,(1,1),padding=0)
 self.relu = nn.ReLU()
self.conv2 = nn.Conv2d(64,16,(1,1),padding=0)
 self.conv3 = nn.Conv2d(16,8,(1,1),padding=0)
 self.conv4 = nn.Conv2d(8,1,(1,1),padding=0)
 self.sig = nn.Sigmoid()
def forward(self, input):
x = self.relu(self.conv1(input))
x = self.relu(self.conv2(x))
x = self.relu(self.conv3(x))
x = self.sig(self.conv4(x))
return x
class Attention_Cell(nn.Module):
def __init__(self,in_channels):
super().__init__()
self.bn2 = nn.BatchNorm2d(in_channels)
 self.basic_cell = Basic_cell(in_channels)
 self.GAP = nn.AdaptiveAvgPool2d(( 1,1))
def forward(self, input):
bn_x = self.bn2(input)
 x = self.basic_cell(bn_x)
 multi_x = bn_x*x
x = self.GAP(x)
 multi_x = self.GAP(multi_x)
 output = torch.div(multi_x,x)
 return output
class RegressionDense(nn.Sequential):
def __init__(self, in_channels):
super().__init__(
 nn.BatchNorm1d(in_channels),
nn.Dropout(0.5),
nn.Linear(in_channels, 1280),
nn.PReLU() ,
nn.BatchNorm1d(1280),
nn.Dropout(0.2),
nn.Linear(1280, 1), 
)
 class ClassfierDense(nn.Sequential):
def __init__(self, in_channels):
super().__init__(
 nn.BatchNorm1d(in_channels),
nn.Dropout(0.5),
nn.Linear(in_channels, 1280),
nn.PReLU() ,
nn.BatchNorm1d(1280),
nn.Dropout(0.2),
nn.Linear(1280, 5), 
)
import os
import cv2
import skimage.io
from tqdm.notebook import tqdm
import zipfile
import numpy as np
import matplotlib.pyplot as plt
import time
from skimage import morphology
from pdb import set_trace
import PIL
import torch
from torch import tensor
from torch.utils.data import DataLoader, Dataset
import pandas as pd
from torch import nn
import fastai
from fastai.vision import *
from joblib import Parallel, delayed<save_to_csv>
class CustomEfficientNet_Att(nn.Module):
def __init__(self, model_name='efficientnet-b0', weights_path=None):
assert model_name in('efficientnet-b0', 'efficientnet-b1', 'efficientnet-b2', 'efficientnet-b3', 'e fficientnet-b4', 'efficientnet-b5')
 super().__init__()
in_channels = 1280
if model_name is 'efficientnet-b0':
in_channels = 1280
if model_name is 'efficientnet-b3':
in_channels = 1536
self.basic_net = EfficientNet.from_name(model_name)
 pretrained_dict = torch.load(weights_path)
 model_dict = self.basic_net.state_dict()
state_dict = {}
for k, v in pretrained_dict.items() :
if k in model_dict.keys() :
state_dict[k] = v
model_dict.update(state_dict)
 self.basic_net.load_state_dict(model_dict)
 self.attention_cell= Attention_Cell(in_channels)
 self.regress = RegressionDense(in_channels)
 self.classfier = ClassfierDense(in_channels)
 def forward(self, x):
x = self.basic_net(x)
 x = self.attention_cell(x)
 x = x.squeeze(-1 ).squeeze(-1)
 pred_1 = self.regress(x)
 pred_2 = self.classfier(x)
 return pred_1,pred_2
SAMPLE = '.. /input/prostate-cancer-grade-assessment/sample_submission.csv'
sub_df = pd.read_csv(SAMPLE)
 sub_df.to_csv('submission.csv', index=False )<define_variables>
class CustomResnet_Att(nn.Module):
def __init__(self, model_name='resnet50', weights_path=None):
assert model_name in('resnet50', 'resnet101', 'resnet152')
 super().__init__()
self.basic_net = pretrainedmodels.__dict__[model_name](pretrained=None)
 pretrained_dict = self.basic_net.load_state_dict(torch.load(weights_path))
model_dict = self.basic_net.state_dict()
state_dict = {}
for k, v in pretrained_dict.items() :
if k in model_dict.keys() :
state_dict[k] = v
model_dict.update(state_dict)
 self.basic_net.load_state_dict(model_dict)
 self.attention_cell= Attention_Cell()
self.regress = RegressionDense(1280)
 self.classfier = ClassfierDense(1280)
 def forward(self, x):
x = self.basic_net(x)
 x = self.attention_cell(x)
 x = x.squeeze(-1 ).squeeze(-1)
 pred_1 = self.regress(x)
 pred_2 = self.classfier(x)
 return pred_1,pred_2
def get_drhb_best(debug=False):
N_FOLDS = 1
sz = 224
bs = 2
N = 81
nworkers = 2
imagenet_stats = [[0.485, 0.456, 0.406], [0.229, 0.224, 0.225]]
DATA = '.. /input/prostate-cancer-grade-assessment/test_images'
TEST = '.. /input/prostate-cancer-grade-assessment/test.csv'
if debug:
DATA = '.. /input/prostate-cancer-grade-assessment/train_images'
TEST = '.. /input/prostate-cancer-grade-assessment/train.csv'
SAMPLE = '.. /input/prostate-cancer-grade-assessment/sample_submission.csv'
MODELS = ['.. /input/exp-80/EXP_80_RESNET_34_TILES_81_SQ_FT_NBN_SE_DUAL_0_sq_features_41.pth']
IMG_OUT = 'imgs'
class OptimizedRounder() :
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
elif pred >= coef[3] and pred < coef[4]:
X_p[i] = 4
else:
X_p[i] = 5
ll = quadratic_weighted_kappa(y, X_p)
 return -ll
def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X=X, y=y)
 initial_coef = [0.5, 1.5, 2.5, 3.5, 4.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 def predict(self, X, coef):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
elif pred >= coef[3] and pred < coef[4]:
X_p[i] = 4
else:
X_p[i] = 5
return X_p
def coefficients(self):
return self.coef_['x']
def prepare_df(df):
uniq_id = list(df['image_id'].unique())
 return df, uniq_id
def tile(img_name):
img = skimage.io.MultiImage(img_name)[-2]
result = []
shape = img.shape
pad0,pad1 =(sz - shape[0]%sz)%sz,(sz - shape[1]%sz)%sz
img = np.pad(img,[[pad0//2,pad0-pad0//2],[pad1//2,pad1-pad1//2],[0,0]],
constant_values=255)
 img = img.reshape(img.shape[0]//sz,sz,img.shape[1]//sz,sz,3)
 img = img.transpose(0,2,1,3,4 ).reshape(-1,sz,sz,3)
 idxs = np.argsort(img.reshape(img.shape[0],-1 ).sum(-1))
img = img[idxs][:N]
if len(img)<N:
n = N-len(img)
 img = np.concatenate([img, np.full(( n,sz,sz,3),255,dtype=np.uint8)],0)
 for i in range(len(img)) :
result.append({'img':img[i], 'idx':i})
 return result
def save_imgs(image_name, folder_path=DATA, image_folder_out=IMG_OUT):
image_nm = folder_path + '/' + image_name + '.tiff'
try:
tiles = tile(image_nm)
 for t in tiles:
img,idx = t['img'], t['idx']
cv2.imwrite(f'{image_folder_out}/{image_name}_{idx}.png', cv2.cvtColor(img, cv2.COLOR_RGB2BGR))
except:
pass
class CustomEnd(nn.Module):
def __init__(self, scaler = SigmoidRange(-1, 6.0)) :
super().__init__()
self.scaler_ = scaler
def forward(self, x):
classif = x[:, :-1]
regress = self.scaler_(x[:, -1])
 return classif, regress
def make_divisible(v, divisor=8, min_value=None):
min_value = min_value or divisor
new_v = max(min_value, int(v + divisor / 2)// divisor * divisor)
 if new_v < 0.9 * v:
new_v += divisor
return new_v
def sigmoid(x, inplace: bool = False):
return x.sigmoid_() if inplace else x.sigmoid()
class SqueezeExcite(nn.Module):
def __init__(self, in_chs, se_ratio=0.25, reduced_base_chs=None,
act_layer=nn.ReLU, gate_fn=sigmoid, divisor=1, **_):
super(SqueezeExcite, self ).__init__()
self.gate_fn = gate_fn
reduced_chs = make_divisible(( reduced_base_chs or in_chs)* se_ratio, divisor)
 self.avg_pool = nn.AdaptiveAvgPool2d(1)
 self.conv_reduce = nn.Conv2d(in_chs, reduced_chs, 1, bias=True)
 self.act1 = act_layer(inplace=True)
 self.conv_expand = nn.Conv2d(reduced_chs, in_chs, 1, bias=True)
 def forward(self, x):
x_se = self.avg_pool(x)
 x_se = self.conv_reduce(x_se)
 x_se = self.act1(x_se)
 x_se = self.conv_expand(x_se)
 x = x * self.gate_fn(x_se)
 return x
class CustomEnd(nn.Module):
def __init__(self, scaler = SigmoidRange(-1, 6.0)) :
super().__init__()
self.scaler_ = scaler
def forward(self, x):
classif = x[:, :-1]
regress = self.scaler_(x[:, -1])
 return classif, regress
class ModelSeDul(nn.Module):
def __init__(self):
super().__init__()
m = fastai.vision.models.resnet34()
self.enc = nn.Sequential(*list(m.children())[:-2])
 nc = list(m.children())[-1].in_features
self.cb = SqueezeExcite(nc)
 self.head = nn.Sequential(AdaptiveConcatPool2d() ,
Flatten() ,
nn.Linear(2*nc,512),
nn.ReLU(inplace=True),
nn.Dropout(0.4),
nn.Linear(512,7),
CustomEnd())
 def forward(self, x):
shape = x.shape
n = shape[1]
x = x.view(-1,shape[2],shape[3],shape[4])
 x = self.enc(x)
 shape = x.shape
x = x.view(-1, n, x.shape[1], x.shape[2], x.shape[3] ).permute(0, 2, 1, 3, 4 ).contiguous().\
view(-1, x.shape[1], x.shape[2] * n, x.shape[3])
 x = x.view(x.shape[0], x.shape[1], x.shape[2]//int(np.sqrt(N)) , -1)
 x = self.cb(x)
 x = self.head(x)
 return x[1]
class PandasDSST2(Dataset):
def __init__(self, fnames, stats=imagenet_stats, N=N, sz=sz, path = IMG_OUT):
self.items = fnames
self.stats = list(map(tensor, stats))
self.sz = sz
self.N =N
self.path = path
def __len__(self):
return len(self.items)
 def __getitem__(self, idx):
imgs = []
fns = [f'{self.path}/{self.items[idx]}_{i}.png' for i in range(self.N)]
for fn in fns:
img = PIL.Image.open(fn ).convert('RGB')
 img = tensor(np.array(img)).float() /255
img = self.normalize(img)
 imgs.append(img)
 return torch.cat(imgs ).reshape(self.N, self.sz, self.sz, 3 ).permute(0, 3, 1, 2), self.items[idx]
def normalize(self, x):
return(x-self.stats[0])/self.stats[1]
models = []
for path in MODELS:
state_dict = torch.load(path,map_location=torch.device('cpu')) ['model']
model = ModelSeDul()
model.load_state_dict(state_dict)
 model.float()
model.eval()
model.cuda()
models.append(model)
 optR = OptimizedRounder()
coefficients = [0.5, 1.5, 2.5, 3.5, 4.5]
def get_preds(N, nm, md_, TTA=False):
print('getting predictions:')
 print(f"{N}")
 ds = PandasDSST2(nm, N=N)
 dl = DataLoader(ds, batch_size=bs, num_workers=nworkers, shuffle=False)
 names,preds = [],[]
if TTA:
with torch.no_grad() :
for x,y in tqdm(dl):
x = x.cuda()
x = torch.stack([x,x.flip(-1),x.flip(-2),x.flip(-1,-2),
x.transpose(-1,-2),x.transpose(-1,-2 ).flip(-1),
x.transpose(-1,-2 ).flip(-2),x.transpose(-1,-2 ).flip(-1,-2)],1)
 x = x.view(-1,N,3,sz,sz)
 p = [model(x)for model in md_]
p = torch.stack(p,1)
 try:
p = p.view(bs, 8*len(md_),-1 ).mean(1 ).cpu()
except:
bs_= p.shape[0]
p = p.view(bs_, 8*len(md_),-1 ).mean(1 ).cpu()
names.append(y)
 preds.append(p)
 else:
with torch.no_grad() :
for x,y in tqdm(dl):
x = x.cuda()
x = x.view(-1,N,3,sz,sz)
 p = [model(x)for model in md_]
p = torch.stack(p,1)
 try:
p = p.view(bs, len(md_),-1 ).mean(1 ).cpu()
except:
bs_= p.shape[0]
p = p.view(bs_, len(md_),-1 ).mean(1 ).cpu()
names.append(y)
 preds.append(p)
 names = np.concatenate(names)
 preds = torch.cat(preds ).numpy().reshape(-1)
 sub_df = pd.DataFrame({'image_id': names, 'isup_grade': preds})
 sub_df.to_csv('submission.csv', index=False)
 return sub_df
sub_df = pd.read_csv(SAMPLE)
 if os.path.exists(DATA):
nm = pd.read_csv(TEST ).image_id.to_list()
if debug:
nm=nm[:100]
!mkdir imgs
Parallel(n_jobs=nworkers )(delayed(save_imgs )(i)for i in tqdm(nm)) ;
sub_df = get_preds(81, nm, models, TTA=True)
 sub_df.to_csv('submission.csv', index=False)
 !rm -rf 'imgs'
sub_df.to_csv('submission.csv', index=False)
 return sub_df 
<train_model>
class ClassifierModule(nn.Sequential):
def __init__(self, n_features):
super().__init__(
 nn.BatchNorm1d(n_features),
nn.Dropout(0.5),
nn.Linear(n_features, n_features),
nn.PReLU() ,
nn.BatchNorm1d(n_features),
nn.Dropout(0.2),
nn.Linear(n_features, 1),
)
DATA = '.. /input/prostate-cancer-grade-assessment/test_images'
if os.path.exists(DATA):
res_drhb = get_drhb_best(False )<set_options>
class CustomResNet(nn.Module):
def __init__(self, model_name='resnet50', weights_path=None):
assert model_name in('resnet50', 'resnet101', 'resnet152')
 super().__init__()
self.net = pretrainedmodels.__dict__[model_name](pretrained=None)
 self.net.load_state_dict(torch.load(weights_path))
n_features = self.net.last_linear.in_features
self.net.avgpool = nn.AdaptiveAvgPool2d(1)
 self.net.last_linear = ClassifierModule(n_features)
 def forward(self, x):
return self.net(x )
torch.cuda.empty_cache()<load_pretrained>
class CustomResNeXt(nn.Module):
def __init__(self, model_name='resnext101_32x4d', weights_path=None):
assert model_name in('resnext101_32x4d', 'resnext101_64x4d')
 super().__init__()
self.net = pretrainedmodels.__dict__[model_name](pretrained=None)
 self.net.load_state_dict(torch.load(weights_path))
n_features = self.net.last_linear.in_features
self.net.avg_pool = nn.AdaptiveAvgPool2d(1)
 self.net.last_linear = ClassifierModule(n_features)
 def forward(self, x):
return self.net(x )
def get_igor_preds(debug=False):
weights = '/kaggle/input/rnx50ws7ep28/fold0_ep28pb2.pt'
def fix_weights(weights):
state_dict = torch.load(weights)['model']
new_state_dict = collections.OrderedDict()
for k, v in state_dict.items() :
name = k[7:]
new_state_dict[name] = v
return new_state_dict
def sigmoid_scale(x, low=-1.0, high=6.0):
return torch.sigmoid(x)*(high-low)+ low
def make_divisible(v, divisor=8, min_value=None):
min_value = min_value or divisor
new_v = max(min_value, int(v + divisor / 2)// divisor * divisor)
 if new_v < 0.9 * v:
new_v += divisor
return new_v
def sigmoid(x, inplace: bool = False):
return x.sigmoid_() if inplace else x.sigmoid()
class SqueezeExcite(nn.Module):
def __init__(self, in_chs, se_ratio=0.25, reduced_base_chs=None,
act_layer=nn.ReLU, gate_fn=sigmoid, divisor=1, **_):
super(SqueezeExcite, self ).__init__()
self.gate_fn = gate_fn
reduced_chs = make_divisible(( reduced_base_chs or in_chs)* se_ratio, divisor)
 self.avg_pool = nn.AdaptiveAvgPool2d(1)
 self.conv_reduce = nn.Conv2d(in_chs, reduced_chs, 1, bias=True)
 self.act1 = act_layer(inplace=True)
 self.conv_expand = nn.Conv2d(reduced_chs, in_chs, 1, bias=True)
 def forward(self, x):
x_se = self.avg_pool(x)
 x_se = self.conv_reduce(x_se)
 x_se = self.act1(x_se)
 x_se = self.conv_expand(x_se)
 x = x * self.gate_fn(x_se)
 return x
class AdaptiveConcatPool2d(nn.Module):
def __init__(self, sz=1):
super().__init__()
self.output_size = sz or 1
self.ap = nn.AdaptiveAvgPool2d(self.output_size)
 self.mp = nn.AdaptiveMaxPool2d(self.output_size)
 def forward(self, x):
return torch.cat([self.mp(x), self.ap(x)], 1)
 class c_Conv2d(nn.Conv2d):
def __init__(self, in_channels, out_channels, kernel_size, stride=1,
padding=0, dilation=1, groups=1, bias=True):
super(c_Conv2d, self ).__init__(in_channels, out_channels, kernel_size, stride,
padding, dilation, groups, bias)
 def forward(self, x):
weight = self.weight
weight_mean = weight.mean(dim=1, keepdim=True ).mean(dim=2,
keepdim=True ).mean(dim=3, keepdim=True)
 weight = weight - weight_mean
std = weight.view(weight.size(0), -1 ).std(dim=1 ).view(-1, 1, 1, 1)+ 1e-5
weight = weight / std.expand_as(weight)
 return F.conv2d(x, weight, self.bias, self.stride,
self.padding, self.dilation, self.groups)
 def c_BatchNorm2d(num_features):
return nn.GroupNorm(num_channels=num_features, num_groups=32)
 class Bottleneck(nn.Module):
expansion = 4
def __init__(self, inplanes, planes, baseWidth, cardinality, stride=1, downsample=None):
super(Bottleneck, self ).__init__()
D = int(math.floor(planes *(baseWidth / 64)))
 C = cardinality
self.conv1 = c_Conv2d(inplanes, D*C, kernel_size=1, stride=1, padding=0, bias=False)
 self.bn1 = c_BatchNorm2d(D*C)
 self.conv2 = c_Conv2d(D*C, D*C, kernel_size=3, stride=stride, padding=1, groups=C, bias=False)
 self.bn2 = c_BatchNorm2d(D*C)
 self.conv3 = c_Conv2d(D*C, planes * 4, kernel_size=1, stride=1, padding=0, bias=False)
 self.bn3 = c_BatchNorm2d(planes * 4)
 self.relu = nn.ReLU(inplace=True)
 self.downsample = downsample
def forward(self, x):
residual = x
out = self.conv1(x)
 out = self.bn1(out)
 out = self.relu(out)
 out = self.conv2(out)
 out = self.bn2(out)
 out = self.relu(out)
 out = self.conv3(out)
 out = self.bn3(out)
 if self.downsample is not None:
residual = self.downsample(x)
 out += residual
out = self.relu(out)
 return out
class ResNeXt(nn.Module):
def __init__(self, baseWidth, cardinality, layers, num_classes):
super(ResNeXt, self ).__init__()
block = Bottleneck
self.cardinality = cardinality
self.baseWidth = baseWidth
self.num_classes = num_classes
self.inplanes = 64
self.output_size = 64
self.conv1 = c_Conv2d(3, 64, 7, 2, 3, bias=False)
 self.bn1 = c_BatchNorm2d(64)
 self.relu = nn.ReLU(inplace=True)
 self.maxpool1 = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
 self.layer1 = self._make_layer(block, 64, layers[0])
 self.layer2 = self._make_layer(block, 128, layers[1], 2)
 self.layer3 = self._make_layer(block, 256, layers[2], 2)
 self.layer4 = self._make_layer(block, 512, layers[3], 2)
 self.avgpool = nn.AvgPool2d(7)
 self.fc = nn.Linear(512 * block.expansion, num_classes)
 for m in self.modules() :
if isinstance(m, nn.Conv2d):
n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
m.weight.data.normal_(0, math.sqrt(2./ n))
elif isinstance(m, nn.BatchNorm2d):
m.weight.data.fill_(1)
 m.bias.data.zero_()
def _make_layer(self, block, planes, blocks, stride=1):
downsample = None
if stride != 1 or self.inplanes != planes * block.expansion:
downsample = nn.Sequential(
 c_Conv2d(self.inplanes, planes * block.expansion,
kernel_size=1, stride=stride, bias=False),
c_BatchNorm2d(planes * block.expansion), 
)
 layers = []
layers.append(block(self.inplanes, planes, self.baseWidth, self.cardinality, stride, downsample))
self.inplanes = planes * block.expansion
for i in range(1, blocks):
layers.append(block(self.inplanes, planes, self.baseWidth, self.cardinality))
return nn.Sequential(*layers)
 def forward(self, x):
x = self.conv1(x)
 x = self.bn1(x)
 x = self.relu(x)
 x = self.maxpool1(x)
 x = self.layer1(x)
 x = self.layer2(x)
 x = self.layer3(x)
 x = self.layer4(x)
 x = self.avgpool(x)
 x = x.view(x.size(0), -1)
 x = self.fc(x)
 return x
class Resnext50_ws(nn.Module):
def __init__(self, baseWidth=4, cardinality=32):
super(Resnext50_ws, self ).__init__()
base_model = ResNeXt(baseWidth, cardinality, [3, 4, 6, 3], 1000)
 self.base_model = nn.Sequential(*list(base_model.children())[:-2])
 nc = list(base_model.children())[-1].in_features
self.conv_block = SqueezeExcite(nc)
 self.head = nn.Sequential(AdaptiveConcatPool2d() ,
nn.Flatten() ,
nn.Linear(2 * nc, 512),
nn.ReLU() ,
nn.Dropout(0.4),
nn.Linear(512, 1, bias=False))
def forward(self, x):
n = x.shape[1]
x = x.view(-1, x.shape[2], x.shape[3], x.shape[4])
 x = self.base_model(x)
 x = x.view(-1, n, x.shape[1], x.shape[2], x.shape[3] ).permute(0, 2, 1, 3, 4 ).contiguous().view(-1, x.shape[1], x.shape[2] * n, x.shape[3])
 x = x.view(x.shape[0], x.shape[1], x.shape[2]//7, -1)
 x = self.conv_block(x)
 x = self.head(x)
 return sigmoid_scale(x)
 def crop(img, tol=254):
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
 mask = gray < tol
img1 = img[:, :, 0][np.ix_(mask.any(1), mask.any(0)) ]
img2 = img[:, :, 1][np.ix_(mask.any(1), mask.any(0)) ]
img3 = img[:, :, 2][np.ix_(mask.any(1), mask.any(0)) ]
img = np.stack([img1, img2, img3], axis=-1)
 _, img = cv2.threshold(img, 0, 255, cv2.THRESH_TOZERO)
 return img
def pad_if_needed(img, tile):
if img.shape[0] % tile != 0 or img.shape[1] % tile != 0:
pad1 = int(np.ceil(img.shape[0] / tile))
pad1 = tile * pad1 - img.shape[0]
pad2 = int(np.ceil(img.shape[1] / tile))
pad2 = tile * pad2 - img.shape[1]
img_pad = np.pad(img,(( pad1, 0),(pad2, 0),(0, 0)) , constant_values=255)
 return img_pad
else:
return img
def windows(img_fn, tiles=49, tile_size=224, overlap=1, threshold=254):
res = skimage.io.MultiImage(img_fn)[-2]
res = crop(res)
 res = pad_if_needed(res, tile_size)
 windows = view_as_windows(res,(tile_size, tile_size, 1),(int(tile_size*overlap), int(tile_size*overlap), 1)) [..., 0].transpose(0, 1, 3, 4, 2)
 windows_orig = windows.reshape(windows.shape[0] * windows.shape[1], windows.shape[2], windows.shape[3], windows.shape[4])
 mean = np.mean(windows_orig, axis=(1, 2, 3))
windows = np.delete(windows_orig, np.where(mean >= threshold), axis=0)
 if windows.shape[0] == 0:
windows = windows_orig
idxs = np.argsort(windows.reshape(windows.shape[0], -1 ).sum(-1)) [:tiles]
if windows.shape[0] < tiles:
windows_pad = np.repeat(255*np.ones_like(np.expand_dims(windows[0], 0)) , tiles - windows.shape[0], axis=0)
 windows = np.concatenate(( windows, windows_pad), axis=0)
 else:
windows = windows[idxs]
return windows
def embed_output(output):
thrs = [0.5, 1.5, 2.5, 3.5, 4.5]
output[output < thrs[0]] = 0
output[(output >= thrs[0])&(output < thrs[1])] = 1
output[(output >= thrs[1])&(output < thrs[2])] = 2
output[(output >= thrs[2])&(output < thrs[3])] = 3
output[(output >= thrs[3])&(output < thrs[4])] = 4
output[output >= thrs[4]] = 5
return output
class TestDataset(Dataset):
def __init__(self, df, path):
self.df = df
self.path = path
def __len__(self):
return len(self.df)
 def __getitem__(self, idx):
row = self.df.iloc[idx]
img_path = '%s%s.tiff' %(self.path, row.image_id)
 image = windows(img_path)
 return torch.tensor(image.transpose(0, 3, 1, 2)).float() /255, row.image_id
model = Resnext50_ws().cuda()
model.load_state_dict(fix_weights(weights))
preds = []
preds_raw = []
names = []
path = '/kaggle/input/prostate-cancer-grade-assessment/test_images/'
submission = pd.read_csv('/kaggle/input/prostate-cancer-grade-assessment/test.csv')
 if debug:
path = '/kaggle/input/prostate-cancer-grade-assessment/train_images/'
submission = pd.read_csv('/kaggle/input/prostate-cancer-grade-assessment/train.csv')[:100]
if os.path.exists(path):
test_dataset = TestDataset(submission, path)
 test_loader = DataLoader(test_dataset, batch_size=1, shuffle=False)
 for i,(images, name)in enumerate(test_loader):
images = images.cuda()
with torch.no_grad() :
y_preds = model(images)
 preds.append(embed_output(y_preds.cpu().numpy()))
preds_raw.append(y_preds.cpu().numpy())
 names.extend(name)
 preds_out = np.concatenate(preds)
 preds_out_raw = np.concatenate(preds_raw ).reshape(-1)
 result = pd.DataFrame({
'image_id':names,
'isup_grade':preds_out_raw
})
 return result<load_pretrained>
class CustomSENet(nn.Module):
def __init__(self, model_name='se_resnet50', weights_path=None):
assert model_name in('senet154', 'se_resnet50', 'se_resnet101', 'se_resnet152', 'se_resnext50_32x4d', 'se_resnext101_32x4d')
 super().__init__()
self.net = pretrainedmodels.__dict__[model_name](pretrained=None)
 self.net.load_state_dict(torch.load(weights_path))
n_features = self.net.last_linear.in_features
self.net.avg_pool = nn.AdaptiveAvgPool2d(1)
 self.net.last_linear = ClassifierModule(n_features)
 def forward(self, x):
return self.net(x )
DATA = '.. /input/prostate-cancer-grade-assessment/test_images'
if os.path.exists(DATA):
res_igor = get_igor_preds(False)
 torch.cuda.empty_cache()
gc.collect()<define_variables>
class CustomEfficientNet(nn.Module):
def __init__(self, model_name='efficientnet-b0', weights_path=None):
assert model_name in('efficientnet-b0', 'efficientnet-b1', 'efficientnet-b2', 'efficientnet-b3', 'e fficientnet-b4', 'efficientnet-b5')
 super().__init__()
self.net = EfficientNet.from_name(model_name)
 self.net.load_state_dict(torch.load(weights_path))
n_features = self.net._fc.in_features
self.net._fc = ClassifierModule(n_features)
 def forward(self, x):
return self.net(x )
def get_prediction_medianse50(debug=False):
model_path= [".. /input/pandamsesmoothse50/se_resnext50_32x4d_fold0_bestOptimQWK.pth",
".. /input/pandamsesmoothse50/se_resnext50_32x4d_fold1_bestOptimQWK.pth",
".. /input/pandamsesmoothse50/se_resnext50_32x4d_fold2_bestOptimQWK.pth",
".. /input/pandamsesmoothse50/se_resnext50_32x4d_fold3_bestOptimQWK.pth",
".. /input/pandamsesmoothse50/se_resnext50_32x4d_fold4_bestOptimQWK.pth"]
if debug:
img_dir=".. /input/prostate-cancer-grade-assessment/train_images/"
df=pd.read_csv(".. /input/prostate-cancer-grade-assessment/train.csv")[:100]
else:
df=pd.read_csv(".. /input/prostate-cancer-grade-assessment/test.csv")
 img_dir=".. /input/prostate-cancer-grade-assessment/test_images/"
def tile(img,sz=128,N=12):
img = img.reshape(img.shape[0]//sz,sz,img.shape[1]//sz,sz,3)
 img = img.transpose(0,2,1,3,4 ).reshape(-1,sz,sz,3)
 if len(img)< N:
img = np.pad(img,[[0,N-len(img)],[0,0],[0,0],[0,0]],constant_values=255)
 idxs = np.argsort(img.reshape(img.shape[0],-1 ).sum(-1)) [:N]
img = img[idxs]
return 1-img.astype(np.float32)/255
def crop_white(image: np.ndarray, value: int = 255)-> np.ndarray:
assert image.shape[2] == 3
assert image.dtype == np.uint8
ys, =(image.min(( 1, 2)) < value ).nonzero()
xs, =(image.min(0 ).min(1)< value ).nonzero()
if len(xs)== 0 or len(ys)== 0:
return image
return image[ys.min() :ys.max() + 1, xs.min() :xs.max() + 1]
class DataSet_Test(object):
def __init__(self,
df,
img_dir,
size,
num_patch,
grid_offset=(0,1/3,2/3),
mean=(0.485, 0.456, 0.406),
std=(0.229, 0.224, 0.225), 
):
self.image_ids=df.image_id.tolist()
self.img_dir=img_dir
self.mean=mean
self.std=std
self.size=size
self.num_patch=num_patch
self.grid_offset=grid_offset
def __getitem__(self, idx):
img_id=self.image_ids[idx]
image=skimage.io.MultiImage(os.path.join(self.img_dir, "{}.tiff".format(img_id)))[1]
image=crop_white(image)
 _, encoded_img = cv2.imencode(".jpg", image,(int(cv2.IMWRITE_JPEG_QUALITY), 100))
image = cv2.imdecode(encoded_img, cv2.IMREAD_UNCHANGED)
 shape = image.shape
pad0, pad1 =(self.size - shape[0] % self.size)% self.size,(self.size - shape[1] % self.size)% self.size
pad_up = pad0 // 2
pad_left = pad1 // 2
all_patches=[]
for offset in self.grid_offset:
pad_up_tmp=pad_up+int(offset*self.size)
 pad_left_tmp=pad_left+int(offset*self.size)
 tmp_img = np.pad(image, [[pad_up_tmp, self.size+pad0 - pad_up_tmp], [pad_left_tmp, self.size+pad1 - pad_left_tmp], [0, 0]],constant_values=255)
 patch=tile(tmp_img,sz=self.size,N=self.num_patch)
 all_patches.append(patch)
 all_patches=np.stack(all_patches,axis=0)
 all_patches=(all_patches-self.mean)/self.std
return torch.tensor(all_patches,dtype=torch.float32 ).permute(0,1,4,2,3),img_id
def __len__(self):
return len(self.image_ids)
 class SEModule(nn.Module):
def __init__(self, channels, reduction):
super(SEModule, self ).__init__()
self.avg_pool = nn.AdaptiveAvgPool2d(1)
 self.fc1 = nn.Conv2d(channels, channels // reduction, kernel_size=1,
padding=0)
 self.relu = nn.ReLU(inplace=True)
 self.fc2 = nn.Conv2d(channels // reduction, channels, kernel_size=1,
padding=0)
 self.sigmoid = nn.Sigmoid()
def forward(self, x):
module_input = x
x = self.avg_pool(x)
 x = self.fc1(x)
 x = self.relu(x)
 x = self.fc2(x)
 x = self.sigmoid(x)
 return module_input * x
class Bottleneck(nn.Module):

 def forward(self, x):
residual = x
out = self.conv1(x)
 out = self.bn1(out)
 out = self.relu(out)
 out = self.conv2(out)
 out = self.bn2(out)
 out = self.relu(out)
 out = self.conv3(out)
 out = self.bn3(out)
 if self.downsample is not None:
residual = self.downsample(x)
 out = self.se_module(out)+ residual
out = self.relu(out)
 return out
class SEBottleneck(Bottleneck):

 expansion = 4
def __init__(self, inplanes, planes, groups, reduction, stride=1,
downsample=None):
super(SEBottleneck, self ).__init__()
self.conv1 = nn.Conv2d(inplanes, planes * 2, kernel_size=1, bias=False)
 self.bn1 = nn.BatchNorm2d(planes * 2)
 self.conv2 = nn.Conv2d(planes * 2, planes * 4, kernel_size=3,
stride=stride, padding=1, groups=groups,
bias=False)
 self.bn2 = nn.BatchNorm2d(planes * 4)
 self.conv3 = nn.Conv2d(planes * 4, planes * 4, kernel_size=1,
bias=False)
 self.bn3 = nn.BatchNorm2d(planes * 4)
 self.relu = nn.ReLU(inplace=True)
 self.se_module = SEModule(planes * 4, reduction=reduction)
 self.downsample = downsample
self.stride = stride
class SEResNetBottleneck(Bottleneck):

 expansion = 4
def __init__(self, inplanes, planes, groups, reduction, stride=1,
downsample=None):
super(SEResNetBottleneck, self ).__init__()
self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False,
stride=stride)
 self.bn1 = nn.BatchNorm2d(planes)
 self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, padding=1,
groups=groups, bias=False)
 self.bn2 = nn.BatchNorm2d(planes)
 self.conv3 = nn.Conv2d(planes, planes * 4, kernel_size=1, bias=False)
 self.bn3 = nn.BatchNorm2d(planes * 4)
 self.relu = nn.ReLU(inplace=True)
 self.se_module = SEModule(planes * 4, reduction=reduction)
 self.downsample = downsample
self.stride = stride
class SEResNeXtBottleneck(Bottleneck):

 expansion = 4
def __init__(self, inplanes, planes, groups, reduction, stride=1,
downsample=None, base_width=4):
super(SEResNeXtBottleneck, self ).__init__()
width = math.floor(planes *(base_width / 64)) * groups
self.conv1 = nn.Conv2d(inplanes, width, kernel_size=1, bias=False,
stride=1)
 self.bn1 = nn.BatchNorm2d(width)
 self.conv2 = nn.Conv2d(width, width, kernel_size=3, stride=stride,
padding=1, groups=groups, bias=False)
 self.bn2 = nn.BatchNorm2d(width)
 self.conv3 = nn.Conv2d(width, planes * 4, kernel_size=1, bias=False)
 self.bn3 = nn.BatchNorm2d(planes * 4)
 self.relu = nn.ReLU(inplace=True)
 self.se_module = SEModule(planes * 4, reduction=reduction)
 self.downsample = downsample
self.stride = stride
class SENet(nn.Module):
def __init__(self, block, layers, groups, reduction, dropout_p=0.2,
inplanes=128, input_3x3=True, downsample_kernel_size=3,
downsample_padding=1, num_classes=1000):

 super(SENet, self ).__init__()
self.inplanes = inplanes
if input_3x3:
layer0_modules = [ 
('conv1', nn.Conv2d(3, 64, 3, stride=2, padding=1,
bias=False)) , 
('bn1', nn.BatchNorm2d(64)) , 
('relu1', nn.ReLU(inplace=True)) , 
('conv2', nn.Conv2d(64, 64, 3, stride=1, padding=1,
bias=False)) , 
('bn2', nn.BatchNorm2d(64)) , 
('relu2', nn.ReLU(inplace=True)) , 
('conv3', nn.Conv2d(64, inplanes, 3, stride=1, padding=1,
bias=False)) , 
('bn3', nn.BatchNorm2d(inplanes)) , 
('relu3', nn.ReLU(inplace=True)) ,
]
else:
layer0_modules = [ 
('conv1', nn.Conv2d(3, inplanes, kernel_size=7, stride=2,
padding=3, bias=False)) , 
('bn1', nn.BatchNorm2d(inplanes)) , 
('relu1', nn.ReLU(inplace=True)) ,
]
layer0_modules.append(( 'pool', nn.MaxPool2d(3, stride=2,
ceil_mode=True)))
 self.layer0 = nn.Sequential(OrderedDict(layer0_modules))
self.layer1 = self._make_layer(
 block,
planes=64,
blocks=layers[0],
groups=groups,
reduction=reduction,
downsample_kernel_size=1,
downsample_padding=0 
)
 self.layer2 = self._make_layer(
 block,
planes=128,
blocks=layers[1],
stride=2,
groups=groups,
reduction=reduction,
downsample_kernel_size=downsample_kernel_size,
downsample_padding=downsample_padding 
)
 self.layer3 = self._make_layer(
 block,
planes=256,
blocks=layers[2],
stride=2,
groups=groups,
reduction=reduction,
downsample_kernel_size=downsample_kernel_size,
downsample_padding=downsample_padding 
)
 self.layer4 = self._make_layer(
 block,
planes=512,
blocks=layers[3],
stride=2,
groups=groups,
reduction=reduction,
downsample_kernel_size=downsample_kernel_size,
downsample_padding=downsample_padding 
)
 self.avg_pool = nn.AvgPool2d(7, stride=1)
 self.dropout = nn.Dropout(dropout_p)if dropout_p is not None else None
self.last_linear = nn.Linear(512 * block.expansion, num_classes)
 def _make_layer(self, block, planes, blocks, groups, reduction, stride=1,
downsample_kernel_size=1, downsample_padding=0):
downsample = None
if stride != 1 or self.inplanes != planes * block.expansion:
downsample = nn.Sequential(
 nn.Conv2d(self.inplanes, planes * block.expansion,
kernel_size=downsample_kernel_size, stride=stride,
padding=downsample_padding, bias=False),
nn.BatchNorm2d(planes * block.expansion), 
)
 layers = []
layers.append(block(self.inplanes, planes, groups, reduction, stride,
downsample))
self.inplanes = planes * block.expansion
for i in range(1, blocks):
layers.append(block(self.inplanes, planes, groups, reduction))
return nn.Sequential(*layers)
 def features(self, x):
x = self.layer0(x)
 x = self.layer1(x)
 x = self.layer2(x)
 x = self.layer3(x)
 x = self.layer4(x)
 return x
def features_ckpt(self,x):
x.requires_grad=True
x = checkpoint(self.layer0,x)
 x = checkpoint_sequential(self.layer1,3,x)
 x = checkpoint_sequential(self.layer2,4,x)
 x = checkpoint_sequential(self.layer3,6,x)
 x = checkpoint_sequential(self.layer4,3,x)
 return x
def logits(self, x):
x = self.avg_pool(x)
 if self.dropout is not None:
x = self.dropout(x)
 x = x.view(x.size(0), -1)
 x = self.last_linear(x)
 return x
def forward(self, x):
x = self.features(x)
 x = self.logits(x)
 return x
def se_resnext50_32x4d(num_classes=1000, pretrained='imagenet'):
model = SENet(SEResNeXtBottleneck, [3, 4, 6, 3], groups=32, reduction=16,
dropout_p=None, inplanes=64, input_3x3=False,
downsample_kernel_size=1, downsample_padding=0,
num_classes=num_classes)
 return model
class AttentionPool(nn.Module):
def __init__(self,in_ch,hidden=512,dropout=True):
super().__init__()
self.in_ch=in_ch
module=[nn.Linear(in_ch,hidden,bias=True),
nn.Tanh()
]
if dropout:
module.append(nn.Dropout(0.25))
module.append(nn.Linear(hidden,1,bias=True))
self.attention=nn.Sequential(*module)
 def forward(self,x):
num_patch=x.size(1)
 x=x.view(-1,x.size(2))
A=self.attention(x)
 A=A.view(-1,num_patch,1)
 wt=F.softmax(A,dim=1)
 return(x.view(-1,num_patch,self.in_ch)*wt ).sum(dim=1),A
class AdaptiveConcatPool2d_Attention(nn.Module):
def __init__(self, in_ch,hidden=512,dropout=True):
super().__init__()
sz =(1,1)
 self.ap = AttentionPool(in_ch,hidden=hidden,dropout=dropout)
 self.mp = nn.AdaptiveMaxPool2d(sz)
 self.in_ch=in_ch
def forward(self, x):
ap,A=self.ap(x)
 mp=torch.max(x,dim=1)[0]
return torch.cat([ap, mp], dim=1),A
class PANDA_Model_Attention_Concat_MultiTask_Headv2(nn.Module):
def __init__(self,arch='se_resnext50_32x4d',dropout=0.25,num_classes=6,checkpoint=False,scale_op=True):
super().__init__()
if "se" in arch:
self.base_model=se_resnext50_32x4d(pretrained=None)
 back_feature=self.base_model.last_linear.in_features
elif "efficientnet" in arch:
self.base_model = EfficientNet.from_pretrained(arch, num_classes=num_classes)
 back_feature = self.base_model._fc.in_features
else:
self.base_model= resnet34(pretrained=None)
 back_feature=self.base_model.last_linear.in_features
self.checkpoint=checkpoint
self.avg_pool=nn.AdaptiveAvgPool2d(1)
 self.scale_op=scale_op
self.attention=AdaptiveConcatPool2d_Attention(in_ch=back_feature,hidden=512,dropout=dropout>0)
 self.label_head=nn.Sequential(
 nn.Dropout(p=dropout),
nn.Linear(back_feature,1,bias=True)
)
 self.reg_head = nn.Sequential(
 nn.Dropout(p=dropout),
nn.Linear(2*back_feature,1,bias=True), 
)
 self.cls_head = nn.Sequential(
 nn.Dropout(p=dropout),
nn.Linear(2*back_feature,num_classes,bias=True), 
)
 def forward(self,x):
B,N,C,H,W=x.shape
x=x.view(-1,C,H,W)
 if self.checkpoint:
x=self.base_model.features_ckpt(x)
 else:
x=self.base_model.features(x)
 x=self.avg_pool(x ).view(x.size(0),-1)
 patch_pred=self.label_head(x)
 x=x.view(B,N,-1)
 x,A=self.attention(x)
 reg_pred=self.reg_head(x ).view(-1)
 if self.scale_op:
reg_pred=7.*torch.sigmoid(reg_pred)-1.
 cls_pred=self.cls_head(x)
 return reg_pred,cls_pred,patch_pred,A
modellist=[]
for path in model_path:
model=PANDA_Model_Attention_Concat_MultiTask_Headv2(arch='se_resnext50_32x4d',dropout=0.4,num_classes=6,scale_op=True)
 model.cuda()
print("Loading",path)
 ckpt=torch.load(path)
 model.load_state_dict(ckpt['state_dict'])
 model.eval()
modellist.append(model)
 print(len(modellist),"models Loaded")
 grid_offset=[0,1/2]
num_offset=len(grid_offset)
 num_patch=48
patch_size=192
dataset=DataSet_Test(df,img_dir,size=patch_size,num_patch=num_patch,grid_offset=grid_offset)
 dataloader=DataLoader(dataset,batch_size=4,shuffle=False,num_workers=2)
 def get_expectation(cls_output,label):
prob=torch.softmax(cls_output.cpu() ,dim=2)
 prob=torch.mean(prob,dim=1)
 return(label*prob ).sum(dim=1)
 label=torch.tensor([0,1,2,3,4,5],dtype=torch.float32 ).view(1,-1)
 reg_weight=1
if os.path.exists(img_dir):
prediction=[]
name=[]
for images,img_ids in tqdm(dataloader):
images=images.cuda()
images=images.view(-1,num_patch,3,patch_size,patch_size)
 reg_pred=0.
 cls_pred=0.
 with torch.no_grad() :
for model in modellist:
reg_output,cls_output,patch_pred,A=model(images)
 cls_output=cls_output.view(-1,num_offset,6)
 reg_output=reg_output.view(-1,num_offset)
 reg_pred+=reg_output.detach().cpu().numpy().mean(axis=1)
 cls_pred+=get_expectation(cls_output,label ).numpy()
reg_pred/=len(modellist)
 cls_pred/=len(modellist)
 pred_batch=reg_weight*reg_pred+(1-reg_weight)*cls_pred
prediction.append(pred_batch)
 name.extend(img_ids)
 prediction=np.concatenate(prediction ).reshape(-1)
 result=pd.DataFrame({
'image_id':name,
'isup_grade':prediction
})
 return result
else:
df=pd.read_csv(".. /input/prostate-cancer-grade-assessment/sample_submission.csv")
 return df 
<find_best_params>
LOGGER.debug(f'Fold: {FOLD}')
 LOGGER.debug(f'Model: {MODEL}')
 LOGGER.debug(f'Train params: {train_params}' )
DATA = '.. /input/prostate-cancer-grade-assessment/test_images'
if os.path.exists(DATA):
res_rs50 = get_prediction_medianse50(debug=False)
 torch.cuda.empty_cache()
gc.collect() 
<define_variables>
with timer('Prepare train and valid sets'):
with timer(' * define dataset'):
train_dataset = MyDataset(train_df, transform =train_transform,train=True)
 valid_dataset = MyDataset(val_df, transform=train_transform,train =False)
 with timer(' * define dataloader'):
train_loader = DataLoader(train_dataset,
batch_size=train_params['train_batch_size'],
shuffle=True)
 valid_loader = DataLoader(valid_dataset,
batch_size=train_params['test_batch_size'],
shuffle=False)
 LOGGER.debug(f'train size: {len(train_dataset)}, valid size: {len(valid_dataset)}' )
torch.backends.cudnn.benchmark = False
model_path_last_level=[]
model_path_this_level=[]
for f in range(5):
model_path_last_level.append(".. /input/panda-sekfold/se_resnext50_32x4d_fold{}_bestOptimQWK.pth".format(f,f))
print("Last level weight")
 print(model_path_last_level)
 model_path_this_level.append(".. /input/panda-efb0-largebs/efficientnet-b0_fold0_bestOptimQWK.pth")
 model_path_this_level.append(".. /input/panda-efb0-largebs/efficientnet-b0_fold1_bestOptimQWK.pth")
 model_path_this_level.append(".. /input/panda-efb0-largebs/efficientnet-b0_fold2_bestOptimQWK.pth")
 model_path_this_level.append(".. /input/panda-efb0-largebs/efficientnet-b0_fold4_bestOptimQWK.pth")
 print("This level weight")
 print(model_path_this_level)
 tiff_dir=".. /input/prostate-cancer-grade-assessment/test_images/"
df=pd.read_csv(".. /input/prostate-cancer-grade-assessment/test.csv")
 class SEModule(nn.Module):
def __init__(self, channels, reduction):
super(SEModule, self ).__init__()
self.avg_pool = nn.AdaptiveAvgPool2d(1)
 self.fc1 = nn.Conv2d(channels, channels // reduction, kernel_size=1,
padding=0)
 self.relu = nn.ReLU(inplace=True)
 self.fc2 = nn.Conv2d(channels // reduction, channels, kernel_size=1,
padding=0)
 self.sigmoid = nn.Sigmoid()
def forward(self, x):
module_input = x
x = self.avg_pool(x)
 x = self.fc1(x)
 x = self.relu(x)
 x = self.fc2(x)
 x = self.sigmoid(x)
 return module_input * x
class Bottleneck(nn.Module):

 def forward(self, x):
residual = x
out = self.conv1(x)
 out = self.bn1(out)
 out = self.relu(out)
 out = self.conv2(out)
 out = self.bn2(out)
 out = self.relu(out)
 out = self.conv3(out)
 out = self.bn3(out)
 if self.downsample is not None:
residual = self.downsample(x)
 out = self.se_module(out)+ residual
out = self.relu(out)
 return out
class SEBottleneck(Bottleneck):

 expansion = 4
def __init__(self, inplanes, planes, groups, reduction, stride=1,
downsample=None):
super(SEBottleneck, self ).__init__()
self.conv1 = nn.Conv2d(inplanes, planes * 2, kernel_size=1, bias=False)
 self.bn1 = nn.BatchNorm2d(planes * 2)
 self.conv2 = nn.Conv2d(planes * 2, planes * 4, kernel_size=3,
stride=stride, padding=1, groups=groups,
bias=False)
 self.bn2 = nn.BatchNorm2d(planes * 4)
 self.conv3 = nn.Conv2d(planes * 4, planes * 4, kernel_size=1,
bias=False)
 self.bn3 = nn.BatchNorm2d(planes * 4)
 self.relu = nn.ReLU(inplace=True)
 self.se_module = SEModule(planes * 4, reduction=reduction)
 self.downsample = downsample
self.stride = stride
class SEResNetBottleneck(Bottleneck):

 expansion = 4
def __init__(self, inplanes, planes, groups, reduction, stride=1,
downsample=None):
super(SEResNetBottleneck, self ).__init__()
self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False,
stride=stride)
 self.bn1 = nn.BatchNorm2d(planes)
 self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, padding=1,
groups=groups, bias=False)
 self.bn2 = nn.BatchNorm2d(planes)
 self.conv3 = nn.Conv2d(planes, planes * 4, kernel_size=1, bias=False)
 self.bn3 = nn.BatchNorm2d(planes * 4)
 self.relu = nn.ReLU(inplace=True)
 self.se_module = SEModule(planes * 4, reduction=reduction)
 self.downsample = downsample
self.stride = stride
class SEResNeXtBottleneck(Bottleneck):

 expansion = 4
def __init__(self, inplanes, planes, groups, reduction, stride=1,
downsample=None, base_width=4):
super(SEResNeXtBottleneck, self ).__init__()
width = math.floor(planes *(base_width / 64)) * groups
self.conv1 = nn.Conv2d(inplanes, width, kernel_size=1, bias=False,
stride=1)
 self.bn1 = nn.BatchNorm2d(width)
 self.conv2 = nn.Conv2d(width, width, kernel_size=3, stride=stride,
padding=1, groups=groups, bias=False)
 self.bn2 = nn.BatchNorm2d(width)
 self.conv3 = nn.Conv2d(width, planes * 4, kernel_size=1, bias=False)
 self.bn3 = nn.BatchNorm2d(planes * 4)
 self.relu = nn.ReLU(inplace=True)
 self.se_module = SEModule(planes * 4, reduction=reduction)
 self.downsample = downsample
self.stride = stride
class SENet(nn.Module):
def __init__(self, block, layers, groups, reduction, dropout_p=0.2,
inplanes=128, input_3x3=True, downsample_kernel_size=3,
downsample_padding=1, num_classes=1000):

 super(SENet, self ).__init__()
self.inplanes = inplanes
if input_3x3:
layer0_modules = [ 
('conv1', nn.Conv2d(3, 64, 3, stride=2, padding=1,
bias=False)) , 
('bn1', nn.BatchNorm2d(64)) , 
('relu1', nn.ReLU(inplace=True)) , 
('conv2', nn.Conv2d(64, 64, 3, stride=1, padding=1,
bias=False)) , 
('bn2', nn.BatchNorm2d(64)) , 
('relu2', nn.ReLU(inplace=True)) , 
('conv3', nn.Conv2d(64, inplanes, 3, stride=1, padding=1,
bias=False)) , 
('bn3', nn.BatchNorm2d(inplanes)) , 
('relu3', nn.ReLU(inplace=True)) ,
]
else:
layer0_modules = [ 
('conv1', nn.Conv2d(3, inplanes, kernel_size=7, stride=2,
padding=3, bias=False)) , 
('bn1', nn.BatchNorm2d(inplanes)) , 
('relu1', nn.ReLU(inplace=True)) ,
]
layer0_modules.append(( 'pool', nn.MaxPool2d(3, stride=2,
ceil_mode=True)))
 self.layer0 = nn.Sequential(OrderedDict(layer0_modules))
self.layer1 = self._make_layer(
 block,
planes=64,
blocks=layers[0],
groups=groups,
reduction=reduction,
downsample_kernel_size=1,
downsample_padding=0 
)
 self.layer2 = self._make_layer(
 block,
planes=128,
blocks=layers[1],
stride=2,
groups=groups,
reduction=reduction,
downsample_kernel_size=downsample_kernel_size,
downsample_padding=downsample_padding 
)
 self.layer3 = self._make_layer(
 block,
planes=256,
blocks=layers[2],
stride=2,
groups=groups,
reduction=reduction,
downsample_kernel_size=downsample_kernel_size,
downsample_padding=downsample_padding 
)
 self.layer4 = self._make_layer(
 block,
planes=512,
blocks=layers[3],
stride=2,
groups=groups,
reduction=reduction,
downsample_kernel_size=downsample_kernel_size,
downsample_padding=downsample_padding 
)
 self.avg_pool = nn.AvgPool2d(7, stride=1)
 self.dropout = nn.Dropout(dropout_p)if dropout_p is not None else None
self.last_linear = nn.Linear(512 * block.expansion, num_classes)
 def _make_layer(self, block, planes, blocks, groups, reduction, stride=1,
downsample_kernel_size=1, downsample_padding=0):
downsample = None
if stride != 1 or self.inplanes != planes * block.expansion:
downsample = nn.Sequential(
 nn.Conv2d(self.inplanes, planes * block.expansion,
kernel_size=downsample_kernel_size, stride=stride,
padding=downsample_padding, bias=False),
nn.BatchNorm2d(planes * block.expansion), 
)
 layers = []
layers.append(block(self.inplanes, planes, groups, reduction, stride,
downsample))
self.inplanes = planes * block.expansion
for i in range(1, blocks):
layers.append(block(self.inplanes, planes, groups, reduction))
return nn.Sequential(*layers)
 def features(self, x):
x = self.layer0(x)
 x = self.layer1(x)
 x = self.layer2(x)
 x = self.layer3(x)
 x = self.layer4(x)
 return x
def features_ckpt(self,x):
x.requires_grad=True
x = checkpoint(self.layer0,x,preserve_rng_state=False)
 x = checkpoint_sequential(self.layer1,3,x,preserve_rng_state=False)
 x = checkpoint_sequential(self.layer2,4,x,preserve_rng_state=False)
 x = checkpoint_sequential(self.layer3,6,x,preserve_rng_state=False)
 x = checkpoint_sequential(self.layer4,3,x,preserve_rng_state=False)
 return x
def logits(self, x):
x = self.avg_pool(x)
 if self.dropout is not None:
x = self.dropout(x)
 x = x.view(x.size(0), -1)
 x = self.last_linear(x)
 return x
def forward(self, x):
x = self.features(x)
 x = self.logits(x)
 return x
def se_resnext50_32x4d(num_classes=1000, pretrained='imagenet'):
model = SENet(SEResNeXtBottleneck, [3, 4, 6, 3], groups=32, reduction=16,
dropout_p=None, inplanes=64, input_3x3=False,
downsample_kernel_size=1, downsample_padding=0,
num_classes=num_classes)
 if pretrained is not None:
settings = pretrained_settings['se_resnext50_32x4d'][pretrained]
initialize_pretrained_model(model, num_classes, settings)
 return model
class ResNet(nn.Module):
def __init__(self, model):
super(ResNet, self ).__init__()
self.conv1 = model.conv1
self.bn1 = model.bn1
self.relu = model.relu
self.maxpool = model.maxpool
self.layer1 = model.layer1
self.layer2 = model.layer2
self.layer3 = model.layer3
self.layer4 = model.layer4
self.avgpool = model.avgpool
self.last_linear = model.fc
def features(self, input):
x = self.conv1(input)
 x = self.bn1(x)
 x = self.relu(x)
 x = self.maxpool(x)
 x = self.layer1(x)
 x = self.layer2(x)
 x = self.layer3(x)
 x = self.layer4(x)
 return x
def features_ckpt(self, input):
x = self.conv1(input)
 x = self.bn1(x)
 x = self.relu(x)
 x = self.maxpool(x)
 x = checkpoint_sequential(self.layer1,3,x)
 x = checkpoint_sequential(self.layer2,4,x)
 x = checkpoint_sequential(self.layer3,6,x)
 x = checkpoint_sequential(self.layer4,3,x)
 return x
def logits(self, features):
x = self.avgpool(features)
 x = x.view(x.size(0), -1)
 x = self.last_linear(x)
 return x
def forward(self, input):
x = self.features(input)
 x = self.logits(x)
 return x
def resnet34(num_classes=1000, pretrained='imagenet'):

 model = models.resnet34(pretrained=False, num_classes=num_classes)
 if pretrained is not None:
settings = pretrained_settings['resnet34'][pretrained]
model = load_pretrained(model, num_classes, settings)
 model = ResNet(model)
 return model

 GlobalParams = collections.namedtuple('GlobalParams', [
'batch_norm_momentum', 'batch_norm_epsilon', 'dropout_rate',
'num_classes', 'width_coefficient', 'depth_coefficient',
'depth_divisor', 'min_depth', 'drop_connect_rate', 'image_size'])
 BlockArgs = collections.namedtuple('BlockArgs', [
'kernel_size', 'num_repeat', 'input_filters', 'output_filters',
'expand_ratio', 'id_skip', 'stride', 'se_ratio'])
 GlobalParams.__new__.__defaults__ =(None,)* len(GlobalParams._fields)
 BlockArgs.__new__.__defaults__ =(None,)* len(BlockArgs._fields)
 class SwishImplementation(torch.autograd.Function):
@staticmethod
def forward(ctx, i):
result = i * torch.sigmoid(i)
 ctx.save_for_backward(i)
 return result
@staticmethod
def backward(ctx, grad_output):
i = ctx.saved_variables[0]
sigmoid_i = torch.sigmoid(i)
 return grad_output *(sigmoid_i *(1 + i *(1 - sigmoid_i)))
 class MemoryEfficientSwish(nn.Module):
def forward(self, x):
return SwishImplementation.apply(x)
 class Swish(nn.Module):
def forward(self, x):
return x * torch.sigmoid(x)
 def round_filters(filters, global_params):

 multiplier = global_params.width_coefficient
if not multiplier:
return filters
divisor = global_params.depth_divisor
min_depth = global_params.min_depth
filters *= multiplier
min_depth = min_depth or divisor
new_filters = max(min_depth, int(filters + divisor / 2)// divisor * divisor)
 if new_filters < 0.9 * filters:
new_filters += divisor
return int(new_filters)
 def round_repeats(repeats, global_params):

 multiplier = global_params.depth_coefficient
if not multiplier:
return repeats
return int(math.ceil(multiplier * repeats))
def drop_connect(inputs, p, training):

 if not training: return inputs
batch_size = inputs.shape[0]
keep_prob = 1 - p
random_tensor = torch.rand([batch_size, 1, 1, 1], dtype=inputs.dtype, device=inputs.device)+keep_prob
binary_tensor = torch.floor(random_tensor)
 output = inputs / keep_prob * binary_tensor
return output
def get_same_padding_conv2d(image_size=None):

 if image_size is None:
return Conv2dDynamicSamePadding
else:
return partial(Conv2dStaticSamePadding, image_size=image_size)
 class Conv2dDynamicSamePadding(nn.Conv2d):

 def __init__(self, in_channels, out_channels, kernel_size, stride=1, dilation=1, groups=1, bias=True):
super().__init__(in_channels, out_channels, kernel_size, stride, 0, dilation, groups, bias)
 self.stride = self.stride if len(self.stride)== 2 else [self.stride[0]] * 2
def forward(self, x):
ih, iw = x.size() [-2:]
kh, kw = self.weight.size() [-2:]
sh, sw = self.stride
oh, ow = math.ceil(ih / sh), math.ceil(iw / sw)
 pad_h = max(( oh - 1)* self.stride[0] +(kh - 1)* self.dilation[0] + 1 - ih, 0)
 pad_w = max(( ow - 1)* self.stride[1] +(kw - 1)* self.dilation[1] + 1 - iw, 0)
 if pad_h > 0 or pad_w > 0:
x = F.pad(x, [pad_w // 2, pad_w - pad_w // 2, pad_h // 2, pad_h - pad_h // 2])
 return F.conv2d(x, self.weight, self.bias, self.stride, self.padding, self.dilation, self.groups)
 class Conv2dStaticSamePadding(nn.Conv2d):

 def __init__(self, in_channels, out_channels, kernel_size, image_size=None, **kwargs):
super().__init__(in_channels, out_channels, kernel_size, **kwargs)
 self.stride = self.stride if len(self.stride)== 2 else [self.stride[0]] * 2
assert image_size is not None
ih, iw = image_size if type(image_size)== list else [image_size, image_size]
kh, kw = self.weight.size() [-2:]
sh, sw = self.stride
oh, ow = math.ceil(ih / sh), math.ceil(iw / sw)
 pad_h = max(( oh - 1)* self.stride[0] +(kh - 1)* self.dilation[0] + 1 - ih, 0)
 pad_w = max(( ow - 1)* self.stride[1] +(kw - 1)* self.dilation[1] + 1 - iw, 0)
 if pad_h > 0 or pad_w > 0:
self.static_padding = nn.ZeroPad2d(( pad_w // 2, pad_w - pad_w // 2, pad_h // 2, pad_h - pad_h // 2))
else:
self.static_padding = Identity()
def forward(self, x):
x = self.static_padding(x)
 x = F.conv2d(x, self.weight, self.bias, self.stride, self.padding, self.dilation, self.groups)
 return x
class Identity(nn.Module):
def __init__(self,):
super(Identity, self ).__init__()
def forward(self, input):
return input
def efficientnet_params(model_name):

 params_dict = {
'efficientnet-b0':(1.0, 1.0, 224, 0.2),
'efficientnet-b1':(1.0, 1.1, 240, 0.2),
'efficientnet-b2':(1.1, 1.2, 260, 0.3),
'efficientnet-b3':(1.2, 1.4, 300, 0.3),
'efficientnet-b4':(1.4, 1.8, 380, 0.4),
'efficientnet-b5':(1.6, 2.2, 456, 0.4),
'efficientnet-b6':(1.8, 2.6, 528, 0.5),
'efficientnet-b7':(2.0, 3.1, 600, 0.5),
'efficientnet-b8':(2.2, 3.6, 672, 0.5),
'efficientnet-l2':(4.3, 5.3, 800, 0.5),
}
return params_dict[model_name]
class BlockDecoder(object):

 @staticmethod
def _decode_block_string(block_string):

 assert isinstance(block_string, str)
 ops = block_string.split('_')
 options = {}
for op in ops:
splits = re.split(r'(\d.*)', op)
 if len(splits)>= 2:
key, value = splits[:2]
options[key] = value
assert(( 's' in options and len(options['s'])== 1)or 
(len(options['s'])== 2 and options['s'][0] == options['s'][1]))
return BlockArgs(
 kernel_size=int(options['k']),
num_repeat=int(options['r']),
input_filters=int(options['i']),
output_filters=int(options['o']),
expand_ratio=int(options['e']),
id_skip=('noskip' not in block_string),
se_ratio=float(options['se'])if 'se' in options else None,
stride=[int(options['s'][0])])
 @staticmethod
def _encode_block_string(block):

 args = [
'r%d' % block.num_repeat,
'k%d' % block.kernel_size,
's%d%d' %(block.strides[0], block.strides[1]),
'e%s' % block.expand_ratio,
'i%d' % block.input_filters,
'o%d' % block.output_filters
]
if 0 < block.se_ratio <= 1:
args.append('se%s' % block.se_ratio)
 if block.id_skip is False:
args.append('noskip')
 return '_'.join(args)
 @staticmethod
def decode(string_list):

 assert isinstance(string_list, list)
 blocks_args = []
for block_string in string_list:
blocks_args.append(BlockDecoder._decode_block_string(block_string))
return blocks_args
@staticmethod
def encode(blocks_args):

 block_strings = []
for block in blocks_args:
block_strings.append(BlockDecoder._encode_block_string(block))
return block_strings
def efficientnet(width_coefficient=None, depth_coefficient=None, dropout_rate=0.2,
drop_connect_rate=0.2, image_size=None, num_classes=1000):

 blocks_args = [
'r1_k3_s11_e1_i32_o16_se0.25', 'r2_k3_s22_e6_i16_o24_se0.25',
'r2_k5_s22_e6_i24_o40_se0.25', 'r3_k3_s22_e6_i40_o80_se0.25',
'r3_k5_s11_e6_i80_o112_se0.25', 'r4_k5_s22_e6_i112_o192_se0.25',
'r1_k3_s11_e6_i192_o320_se0.25',
]
blocks_args = BlockDecoder.decode(blocks_args)
 global_params = GlobalParams(
 batch_norm_momentum=0.99,
batch_norm_epsilon=1e-3,
dropout_rate=dropout_rate,
drop_connect_rate=drop_connect_rate,
num_classes=num_classes,
width_coefficient=width_coefficient,
depth_coefficient=depth_coefficient,
depth_divisor=8,
min_depth=None,
image_size=image_size, 
)
 return blocks_args, global_params
def get_model_params(model_name, override_params):

 if model_name.startswith('efficientnet'):
w, d, s, p = efficientnet_params(model_name)
 blocks_args, global_params = efficientnet(
 width_coefficient=w, depth_coefficient=d, dropout_rate=p, image_size=s)
 else:
raise NotImplementedError('model name is not pre-defined: %s' % model_name)
 if override_params:
global_params = global_params._replace(**override_params)
 return blocks_args, global_params
url_map = {
'efficientnet-b0': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/efficientnet-b0-355c32eb.pth',
'efficientnet-b1': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/efficientnet-b1-f1951068.pth',
'efficientnet-b2': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/efficientnet-b2-8bb594d6.pth',
'efficientnet-b3': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/efficientnet-b3-5fb5a3c3.pth',
'efficientnet-b4': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/efficientnet-b4-6ed6700e.pth',
'efficientnet-b5': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/efficientnet-b5-b6417697.pth',
'efficientnet-b6': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/efficientnet-b6-c76e70fd.pth',
'efficientnet-b7': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/efficientnet-b7-dcc49843.pth',
}
url_map_advprop = {
'efficientnet-b0': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/adv-efficientnet-b0-b64d5a18.pth',
'efficientnet-b1': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/adv-efficientnet-b1-0f3ce85a.pth',
'efficientnet-b2': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/adv-efficientnet-b2-6e9d97e5.pth',
'efficientnet-b3': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/adv-efficientnet-b3-cdd7c0f4.pth',
'efficientnet-b4': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/adv-efficientnet-b4-44fb3a87.pth',
'efficientnet-b5': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/adv-efficientnet-b5-86493f6b.pth',
'efficientnet-b6': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/adv-efficientnet-b6-ac80338e.pth',
'efficientnet-b7': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/adv-efficientnet-b7-4652b6dd.pth',
'efficientnet-b8': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/adv-efficientnet-b8-22a8fe65.pth',
}
def load_pretrained_weights(model, model_name, load_fc=True, advprop=False):

 url_map_ = url_map_advprop if advprop else url_map
state_dict = model_zoo.load_url(url_map_[model_name])
 if load_fc:
model.load_state_dict(state_dict)
 else:
state_dict.pop('_fc.weight')
 state_dict.pop('_fc.bias')
 res = model.load_state_dict(state_dict, strict=False)
 assert set(res.missing_keys)== set(['_fc.weight', '_fc.bias']), 'issue loading pretrained weights'
print('Loaded pretrained weights for {}'.format(model_name))
class AttentionPool(nn.Module):
def __init__(self,in_ch,hidden=512,dropout=True):
super().__init__()
self.in_ch=in_ch
module=[nn.Linear(in_ch,hidden,bias=True),
nn.Tanh()
]
if dropout:
module.append(nn.Dropout(0.25))
module.append(nn.Linear(hidden,1,bias=True))
self.attention=nn.Sequential(*module)
 def forward(self,x):
num_patch=x.size(1)
 x=x.view(-1,x.size(2))
A=self.attention(x)
 A=A.view(-1,num_patch,1)
 wt=F.softmax(A,dim=1)
 return(x.view(-1,num_patch,self.in_ch)*wt ).sum(dim=1),A
class AdaptiveConcatPool2d_Attention(nn.Module):
def __init__(self, in_ch,hidden=512,dropout=True):
super().__init__()
sz =(1,1)
 self.ap = AttentionPool(in_ch,hidden=hidden,dropout=dropout)
 self.mp = nn.AdaptiveMaxPool2d(sz)
 self.in_ch=in_ch
def forward(self, x):
ap,A=self.ap(x)
 mp=torch.max(x,dim=1)[0]
return torch.cat([ap, mp], dim=1),A
class PANDA_Model_Attention_Concat_MultiTask_Headv2(nn.Module):
def __init__(self,arch='se_resnext50_32x4d',dropout=0.25,num_classes=6,checkpoint=False,scale_op=True):
super().__init__()
self.scale_op=scale_op
if "se" in arch:
self.base_model=se_resnext50_32x4d(pretrained=None)
 back_feature=self.base_model.last_linear.in_features
elif "efficientnet" in arch:
self.base_model = EfficientNet.from_pretrained(arch, num_classes=num_classes)
 back_feature = self.base_model._fc.in_features
else:
self.base_model= resnet34(pretrained=None)
 back_feature=self.base_model.last_linear.in_features
self.checkpoint=checkpoint
self.avg_pool=nn.AdaptiveAvgPool2d(1)
 self.attention=AdaptiveConcatPool2d_Attention(in_ch=back_feature,hidden=512,dropout=dropout>0)
 self.label_head=nn.Sequential(
 nn.Dropout(p=dropout),
nn.Linear(back_feature,1,bias=True)
)
 self.reg_head = nn.Sequential(
 nn.Dropout(p=dropout),
nn.Linear(2*back_feature,1,bias=True), 
)
 self.cls_head = nn.Sequential(
 nn.Dropout(p=dropout),
nn.Linear(2*back_feature,num_classes,bias=True), 
)
 def forward(self,x):
B,N,C,H,W=x.shape
x=x.view(-1,C,H,W)
 if self.checkpoint:
x=self.base_model.features_ckpt(x)
 else:
x=self.base_model.features(x)
 x=self.avg_pool(x ).view(x.size(0),-1)
 patch_pred=self.label_head(x)
 x=x.view(B,N,-1)
 x,A=self.attention(x)
 reg_pred=self.reg_head(x ).view(-1)
 if self.scale_op:
reg_pred=7.*torch.sigmoid(reg_pred)-1.
 cls_pred=self.cls_head(x)
 return reg_pred,cls_pred,patch_pred,A
def crop_white(image, value=255):
assert image.shape[2] == 3
assert image.dtype == np.uint8
ys, =(image.min(( 1, 2)) < value ).nonzero()
xs, =(image.min(0 ).min(1)< value ).nonzero()
if len(xs)== 0 or len(ys)== 0:
return image,np.array([0,0], dtype=np.int)
 return image[ys.min() :ys.max() + 1, xs.min() :xs.max() + 1],np.array([ys.min() , xs.min() ], dtype=np.int)
 def crop_patches(img,bg_threshold, sz=192):
W=img.shape[1]
img = img.reshape(img.shape[0] // sz, sz, img.shape[1] // sz, sz, 3)
 img = img.transpose(0, 2, 1, 3, 4 ).reshape(-1, sz * sz, 3)
 sat = cv2.cvtColor(img, cv2.COLOR_RGB2HSV)[:, :, 1]
background_ratio =(sat < 20 ).astype(np.float32 ).reshape(img.shape[0], -1 ).sum(1)/(sz * sz)
 fg_idx = np.where(background_ratio < bg_threshold)[0]
img = img[fg_idx]
coord = np.stack([fg_idx //(W // sz), fg_idx %(W // sz)], axis=1)*sz
return img.reshape(-1, sz, sz, 3), coord, background_ratio[fg_idx]
def crop_upper_level(img_id, coords, sz, scale=0.5):
image = openslide.OpenSlide(os.path.join(tiff_dir,"{}.tiff".format(img_id)))
 patches = []
for coord in coords:
x = coord[1] * 4
y = coord[0] * 4
x = max(0,x)
 y = max(0,y)
 region_sz = int(sz * 4)
 patch = image.read_region(( x, y), 0,(region_sz, region_sz))
patch = np.asarray(patch.convert("RGB"))
if scale != 1:
patch = cv2.resize(patch, dsize=(int(scale * patch.shape[1]), int(scale * patch.shape[0])))
 patches.append(patch)
 return patches
def get_next_level_patches(img_id,attention,coords,sz=192,scale=0.5,max_patch=64):
idx=np.argsort(attention)[::-1]
N=min(max_patch,len(idx))
idx=idx[:N]
coords=coords[idx]
next_level_patches=crop_upper_level(img_id,coords,sz,scale)
 return next_level_patches,attention[idx],idx
def prepare_next_level_input(next_level_patches,patch_num=32,crop_func=None,
mean=(0.485, 0.456, 0.406),
std=(0.229, 0.224, 0.225),):
next_level_patches=np.stack(next_level_patches,axis=0)
 patches=[]
for patch in next_level_patches:
_, encoded_img = cv2.imencode(".jpg", patch,(int(cv2.IMWRITE_JPEG_QUALITY), 95))
patch = cv2.imdecode(encoded_img, cv2.IMREAD_UNCHANGED)
 patches.append(patch)
 if crop_func is not None:
patches=[crop_func(image=x)['image'] for x in patches]
patches=np.stack(patches,axis=0)
 if len(patches)<patch_num:
patches=np.pad(patches,[[0,patch_num-len(patches)],[0,0],[0,0],[0,0]],constant_values=255)
 patches = 1.0 - patches.astype(np.float32)/ 255
patches =(patches - mean)/std
return torch.tensor(patches,dtype=torch.float32,device='cuda' ).permute(0,3,1,2 ).unsqueeze(0)
 class MBConvBlock(nn.Module):

 def __init__(self, block_args, global_params):
super().__init__()
self._block_args = block_args
self._bn_mom = 1 - global_params.batch_norm_momentum
self._bn_eps = global_params.batch_norm_epsilon
self.has_se =(self._block_args.se_ratio is not None)and(0 < self._block_args.se_ratio <= 1)
 self.id_skip = block_args.id_skip
Conv2d = get_same_padding_conv2d(image_size=global_params.image_size)
 inp = self._block_args.input_filters
oup = self._block_args.input_filters * self._block_args.expand_ratio
if self._block_args.expand_ratio != 1:
self._expand_conv = Conv2d(in_channels=inp, out_channels=oup, kernel_size=1, bias=False)
 self._bn0 = nn.BatchNorm2d(num_features=oup, momentum=self._bn_mom, eps=self._bn_eps)
 k = self._block_args.kernel_size
s = self._block_args.stride
self._depthwise_conv = Conv2d(
 in_channels=oup, out_channels=oup, groups=oup,
kernel_size=k, stride=s, bias=False)
 self._bn1 = nn.BatchNorm2d(num_features=oup, momentum=self._bn_mom, eps=self._bn_eps)
 if self.has_se:
num_squeezed_channels = max(1, int(self._block_args.input_filters * self._block_args.se_ratio))
self._se_reduce = Conv2d(in_channels=oup, out_channels=num_squeezed_channels, kernel_size=1)
 self._se_expand = Conv2d(in_channels=num_squeezed_channels, out_channels=oup, kernel_size=1)
 final_oup = self._block_args.output_filters
self._project_conv = Conv2d(in_channels=oup, out_channels=final_oup, kernel_size=1, bias=False)
 self._bn2 = nn.BatchNorm2d(num_features=final_oup, momentum=self._bn_mom, eps=self._bn_eps)
 self._swish = MemoryEfficientSwish()
def forward(self, inputs, drop_connect_rate=None):

 x = inputs
if self._block_args.expand_ratio != 1:
x = self._swish(self._bn0(self._expand_conv(inputs)))
 x = self._swish(self._bn1(self._depthwise_conv(x)))
 if self.has_se:
x_squeezed = F.adaptive_avg_pool2d(x, 1)
 x_squeezed = self._se_expand(self._swish(self._se_reduce(x_squeezed)))
 x = torch.sigmoid(x_squeezed)* x
x = self._bn2(self._project_conv(x))
input_filters, output_filters = self._block_args.input_filters, self._block_args.output_filters
if self.id_skip and self._block_args.stride == 1 and input_filters == output_filters:
if drop_connect_rate:
x = drop_connect(x, p=drop_connect_rate, training=self.training)
 x = x + inputs
return x
def set_swish(self, memory_efficient=True):

 self._swish = MemoryEfficientSwish() if memory_efficient else Swish()
class EfficientNet(nn.Module):

 def __init__(self, blocks_args=None, global_params=None):
super().__init__()
assert isinstance(blocks_args, list), 'blocks_args should be a list'
assert len(blocks_args)> 0, 'block args must be greater than 0'
self._global_params = global_params
self._blocks_args = blocks_args
Conv2d = get_same_padding_conv2d(image_size=global_params.image_size)
 bn_mom = 1 - self._global_params.batch_norm_momentum
bn_eps = self._global_params.batch_norm_epsilon
in_channels = 3
out_channels = round_filters(32, self._global_params)
 self._conv_stem = Conv2d(in_channels, out_channels, kernel_size=3, stride=2, bias=False)
 self._bn0 = nn.BatchNorm2d(num_features=out_channels, momentum=bn_mom, eps=bn_eps)
 self._blocks = nn.ModuleList([])
 for block_args in self._blocks_args:
block_args = block_args._replace(
 input_filters=round_filters(block_args.input_filters, self._global_params),
output_filters=round_filters(block_args.output_filters, self._global_params),
num_repeat=round_repeats(block_args.num_repeat, self._global_params)
)
 self._blocks.append(MBConvBlock(block_args, self._global_params))
if block_args.num_repeat > 1:
block_args = block_args._replace(input_filters=block_args.output_filters, stride=1)
 for _ in range(block_args.num_repeat - 1):
self._blocks.append(MBConvBlock(block_args, self._global_params))
in_channels = block_args.output_filters
out_channels = round_filters(1280, self._global_params)
 self._conv_head = Conv2d(in_channels, out_channels, kernel_size=1, bias=False)
 self._bn1 = nn.BatchNorm2d(num_features=out_channels, momentum=bn_mom, eps=bn_eps)
 self._avg_pooling = nn.AdaptiveAvgPool2d(1)
 self._dropout = nn.Dropout(self._global_params.dropout_rate)
 self._fc = nn.Linear(out_channels, self._global_params.num_classes)
 self._swish = MemoryEfficientSwish()
def set_swish(self, memory_efficient=True):

 self._swish = MemoryEfficientSwish() if memory_efficient else Swish()
for block in self._blocks:
block.set_swish(memory_efficient)
 def features(self, inputs):

 x = self._swish(self._bn0(self._conv_stem(inputs)))
 for idx, block in enumerate(self._blocks):
drop_connect_rate = self._global_params.drop_connect_rate
if drop_connect_rate:
drop_connect_rate *= float(idx)/ len(self._blocks)
 x = block(x, drop_connect_rate=drop_connect_rate)
 x = self._swish(self._bn1(self._conv_head(x)))
 return x
def features_ckpt(self, inputs):

 x = self._swish(self._bn0(self._conv_stem(inputs)))
 for idx, block in enumerate(self._blocks):
drop_connect_rate = self._global_params.drop_connect_rate
if drop_connect_rate:
drop_connect_rate *= float(idx)/ len(self._blocks)
 x=checkpoint(block,x,torch.tensor(drop_connect_rate ).cuda() ,preserve_rng_state=True)
 x = self._swish(self._bn1(self._conv_head(x)))
 return x
def forward(self, inputs):

 bs = inputs.size(0)
 x = self.features(inputs)
 x = self._avg_pooling(x)
 x = x.view(bs, -1)
 x = self._dropout(x)
 x = self._fc(x)
 return x
@classmethod
def from_name(cls, model_name, override_params=None):
cls._check_model_name_is_valid(model_name)
 blocks_args, global_params = get_model_params(model_name, override_params)
 return cls(blocks_args, global_params)
 @classmethod
def from_pretrained(cls, model_name, advprop=False, num_classes=1000, in_channels=3):
model = cls.from_name(model_name, override_params={'num_classes': num_classes})
 if in_channels != 3:
Conv2d = get_same_padding_conv2d(image_size = model._global_params.image_size)
 out_channels = round_filters(32, model._global_params)
 model._conv_stem = Conv2d(in_channels, out_channels, kernel_size=3, stride=2, bias=False)
 return model
@classmethod
def get_image_size(cls, model_name):
cls._check_model_name_is_valid(model_name)
 _, _, res, _ = efficientnet_params(model_name)
 return res
@classmethod
def _check_model_name_is_valid(cls, model_name):

 valid_models = ['efficientnet-b'+str(i)for i in range(9)]
if model_name not in valid_models:
raise ValueError('model_name should be one of: ' + ', '.join(valid_models))
class PANDAPatchExtraction_Test(object):
def __init__(self,
df,
tiff_dir,
patch_size=192,
bg_threshold=0.8,
trail_offset=[0,1/2],
mean=(0.485, 0.456, 0.406),
std=(0.229, 0.224, 0.225), 
):
self.image_ids = df['image_id'].tolist()
self.tiff_dir = tiff_dir
self.patch_size = patch_size
self.bg_threshold = bg_threshold
self.trail_offset=trail_offset
self.mean = np.array(mean, dtype=np.float32)
 self.std = np.array(std, dtype=np.float)
 def __getitem__(self, idx):
img_id = self.image_ids[idx]
image = skimage.io.MultiImage(os.path.join(self.tiff_dir, "{}.tiff".format(img_id)))[1]
image,offset = crop_white(image)
 _, encoded_img = cv2.imencode(".jpg", image,(int(cv2.IMWRITE_JPEG_QUALITY), 100))
image = cv2.imdecode(encoded_img, cv2.IMREAD_UNCHANGED)
 shape = image.shape
pad0 =(self.patch_size - shape[0] % self.patch_size)% self.patch_size
pad1 =(self.patch_size - shape[1] % self.patch_size)% self.patch_size
pad_up = pad0 // 2
pad_left = pad1 // 2
best_mean_bg_ratio=1000
best_patch=None
best_coord=None
best_pad_offset=None
for trail_offset in self.trail_offset:
trail_pad_up = pad_up + int(self.patch_size * trail_offset)
 trail_pad_left = pad_left + int(self.patch_size * trail_offset)
 image_tmp = np.pad(image, [[trail_pad_up, pad0+self.patch_size - trail_pad_up], [trail_pad_left, pad1+self.patch_size - trail_pad_left], [0, 0]], constant_values=255)
 patches, coords,bg_ratio = crop_patches(image_tmp,self.bg_threshold, sz=self.patch_size)
 if np.mean(bg_ratio)<best_mean_bg_ratio:
best_mean_bg_ratio=np.mean(bg_ratio)
 best_patch=patches
best_coord=coords
best_pad_offset=(trail_pad_up,trail_pad_left)
 offset[0] -= best_pad_offset[0]
offset[1] -= best_pad_offset[1]
if len(best_patch)==0:
best_patch=255*np.ones(( 1,self.patch_size,self.patch_size,3))
best_coord=np.zeros(( 1,2))
best_coord = best_coord + offset.reshape(1, 2)
 best_patch = 1.0 - best_patch.astype(np.float32)/ 255
best_patch =(best_patch - self.mean)/ self.std
return torch.tensor(best_patch, dtype=torch.float32 ).permute(0, 3, 1, 2), best_coord, img_id
def __len__(self):
return len(self.image_ids)
 def safe_run(model,images,max_bs=128):
num_patch=images.shape[1]
split_dim=[max_bs]*(num_patch//max_bs)
 if num_patch%max_bs>0:
split_dim+=[num_patch%max_bs]
attention=[]
for split_img in torch.split(images,split_dim,dim=1):
with torch.no_grad() :
split_img=split_img.cuda()
output,_,_,A=model(split_img)
 attention.append(A.cpu())
 return torch.cat(attention,dim=1)
 model_last_level=[]
model_this_level=[]
for p in model_path_last_level:
print("Loading last level",p)
 model=PANDA_Model_Attention_Concat_MultiTask_Headv2(arch='se_resnext50_32x4d',
dropout=0.25,
num_classes=6,
scale_op=False, 
)
 model.cuda()
ckpt=torch.load(p)
 model.load_state_dict(ckpt['state_dict'])
 model.eval()
model_last_level.append(model)
 for p in model_path_this_level:
print("Loading this level",p)
 model=PANDA_Model_Attention_Concat_MultiTask_Headv2(arch='efficientnet-b0',
dropout=0.4,
num_classes=6,
scale_op=True, 
)
 model.cuda()
ckpt=torch.load(p)
 model.load_state_dict(ckpt['state_dict'])
 model.eval()
model_this_level.append(model)
 print(len(model_last_level),"Median resolution models")
 print(len(model_this_level),"High resolution models")
 dataset=PANDAPatchExtraction_Test(df,
tiff_dir,
patch_size=192,
bg_threshold=0.95,
trail_offset=[0,1/2], 
)
 dataloader=DataLoader(dataset,batch_size=1,shuffle=False,num_workers=2)
 coef=[0.5,1.5,2.5,3.5,4.5]
def predict(X, coef):
return pd.cut(X, [-np.inf] + list(np.sort(coef)) + [np.inf], labels=[0, 1, 2, 3,4,5])
 if os.path.exists(tiff_dir):
prediction=[]
name=[]
for images,coords,img_ids in tqdm(dataloader):
with torch.no_grad() :
A=[]
for model in model_last_level:
att=safe_run(model,images ).view(-1 ).numpy()
A.append(att)
 A=np.stack(A,axis=0 ).mean(axis=0)
 coords=coords.numpy().reshape(-1,2)
 next_level_patches,attention,idx=get_next_level_patches(img_ids[0],A,coords,sz=192,scale=0.5,max_patch=36)
 next_level_patches=prepare_next_level_input(next_level_patches,patch_num=36,crop_func=None)
 reg_pred = 0.
 cls_pred = 0.
 next_level_patches = torch.cat(
 [next_level_patches,next_level_patches.flip(( 3,4)) ],dim=0)
 for model in model_this_level:
reg_output, cls_output, patch_pred, A = model(next_level_patches)
 cls_output = cls_output.view(-1, 6 ).mean(dim=0)
 reg_output = reg_output.view(-1 ).mean(dim=0)
 reg_pred += reg_output.cpu().numpy()
cls_pred += cls_output.cpu().numpy()
reg_pred /= len(model_this_level)
 cls_pred /= len(model_this_level)
 prediction.append(reg_pred)
 name.extend(img_ids)
 prediction=np.array(prediction ).reshape(-1)
 res_ruef=pd.DataFrame({
'image_id':name,
'isup_grade':prediction
})
 print(res_ruef.head())
 else:
df=pd.read_csv(".. /input/prostate-cancer-grade-assessment/sample_submission.csv")
 df.to_csv("./submission.csv",index=False )<set_options>
def multi_loss(pred_reg,pred_classfier,y_true,multi_method='mse_ce',alpha = 1,beta =1):
assert multi_method in('mse_ce','')
 loss = 0;c1=0;c2=0
if multi_method =='mse_ce':
c1 = nn.MSELoss()
c2 = nn.CrossEntropyLoss()
loss_reg = c1(pred_reg,y_true)
 y_true = y_true.long()
loss_class = c2(pred_classfier,y_true.squeeze())
 loss = alpha*loss_reg + beta*loss_class
return loss
del dataloader
del dataset
del model_last_level
del model_this_level
torch.cuda.empty_cache()
gc.collect()<import_modules>
<predict_on_test><EOS> 
. softmax(input, self.dim, _stacklevel=5)
 2020-07-02 02:27:20,329 DEBUG Epoch 1 - avg_train_loss: 0.3125 avg_val_loss: 4.7410 time: 112s
2020-07-02 02:27:20,330 DEBUG - qwk: 0.000000 coefficients: [0.5 1.5 2.5 3.5]
2020-07-02 02:29:02,828 DEBUG Epoch 2 - avg_train_loss: 0.3046 avg_val_loss: 5.4275 time: 102s
2020-07-02 02:29:02,830 DEBUG - qwk: 0.271479 coefficients: [-0.542408 2.047389 3.62589 5.926311]
2020-07-02 02:30:45,871 DEBUG Epoch 3 - avg_train_loss: 0.3203 avg_val_loss: 4.9421 time: 103s
2020-07-02 02:30:45,872 DEBUG - qwk: 0.060628 coefficients: [0.5 1.5 2.5 3.5]
2020-07-02 02:32:29,153 DEBUG Epoch 4 - avg_train_loss: 0.2932 avg_val_loss: 2.9958 time: 103s
2020-07-02 02:32:29,155 DEBUG - qwk: 0.602941 coefficients: [0.276694 1.188741 0.554224 7.733649]
2020-07-02 02:34:12,737 DEBUG Epoch 5 - avg_train_loss: 0.2489 avg_val_loss: 3.6941 time: 104s
2020-07-02 02:34:12,741 DEBUG - qwk: 0.364468 coefficients: [0.40734 0.359478 2.882061 4.309653]
2020-07-02 02:35:55,522 DEBUG Epoch 6 - avg_train_loss: 0.2240 avg_val_loss: 2.6687 time: 103s
2020-07-02 02:35:55,523 DEBUG - qwk: 0.607598 coefficients: [0.494824 0.815553 2.177532 4.449734]
2020-07-02 02:37:39,348 DEBUG Epoch 7 - avg_train_loss: 0.1965 avg_val_loss: 4.0795 time: 104s
2020-07-02 02:37:39,349 DEBUG - qwk: 0.653501 coefficients: [0.596621 1.498057 1.842185 4.625971]
2020-07-02 02:39:22,939 DEBUG Epoch 8 - avg_train_loss: 0.1642 avg_val_loss: 2.6657 time: 104s
2020-07-02 02:39:22,942 DEBUG - qwk: 0.713007 coefficients: [0.461684 1.598026 1.734047 4.394338]
2020-07-02 02:41:06,369 DEBUG Epoch 9 - avg_train_loss: 0.1579 avg_val_loss: 2.8600 time: 103s
2020-07-02 02:41:06,371 DEBUG - qwk: 0.635940 coefficients: [0.519184 1.490187 2.401873 3.643743]
2020-07-02 02:42:49,061 DEBUG Epoch 10 - avg_train_loss: 0.1606 avg_val_loss: 2.5784 time: 103s
2020-07-02 02:42:49,063 DEBUG - qwk: 0.563788 coefficients: [0.516132 0.831461 2.752982 3.96373 ]
2020-07-02 02:44:32,402 DEBUG Epoch 11 - avg_train_loss: 0.1453 avg_val_loss: 2.4899 time: 103s
2020-07-02 02:44:32,404 DEBUG - qwk: 0.559555 coefficients: [0.602128 0.988473 2.593635 3.840373]
2020-07-02 02:46:15,584 DEBUG Epoch 12 - avg_train_loss: 0.1299 avg_val_loss: 2.4178 time: 103s
2020-07-02 02:46:15,586 DEBUG - qwk: 0.618733 coefficients: [0.715129 1.184878 1.110287 4.461294]
2020-07-02 02:46:15,646 INFO [Train model] done in 1247 s .