Kaggle competition: SMEMI309 - Final evaluation challenge 2020
import torch
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from matplotlib import style
import seaborn as sns
from sklearn.model_selection import StratifiedKFold
from joblib import load, dump
from sklearn.metrics import cohen_kappa_score
from sklearn.metrics import confusion_matrix
from fastai import *
from fastai.vision import *
from fastai.callbacks import *
from torchvision import models as md
from torch import nn
from torch.nn import functional as F
import re
import math
import collections
from functools import partial
from torch.utils import model_zoo
from sklearn import metrics
from collections import Counter
import json<categorify>
paths= ".. /input/SMEMI309-final-evaluation-challenge-2020/"
at4_test_path= paths+"AT4_test.csv"
at4_train_path = paths+"AT4_train.csv"
at2_test_path= paths+"AT2_test.csv"
at2_train_path = paths+"AT2_train.csv"
bt_test_path=paths+"BT_test.csv"
bt_train_path=paths+"BT_train.csv"
GlobalParams = collections.namedtuple('GlobalParams', [
'batch_norm_momentum', 'batch_norm_epsilon', 'dropout_rate',
'num_classes', 'width_coefficient', 'depth_coefficient',
'depth_divisor', 'min_depth', 'drop_connect_rate', 'image_size'])
 BlockArgs = collections.namedtuple('BlockArgs', [
'kernel_size', 'num_repeat', 'input_filters', 'output_filters',
'expand_ratio', 'id_skip', 'stride', 'se_ratio'])
 GlobalParams.__new__.__defaults__ =(None,)* len(GlobalParams._fields)
 BlockArgs.__new__.__defaults__ =(None,)* len(BlockArgs._fields)
 def relu_fn(x):

 return x * torch.sigmoid(x)
 def round_filters(filters, global_params):

 multiplier = global_params.width_coefficient
if not multiplier:
return filters
divisor = global_params.depth_divisor
min_depth = global_params.min_depth
filters *= multiplier
min_depth = min_depth or divisor
new_filters = max(min_depth, int(filters + divisor / 2)// divisor * divisor)
 if new_filters < 0.9 * filters:
new_filters += divisor
return int(new_filters)
 def round_repeats(repeats, global_params):

 multiplier = global_params.depth_coefficient
if not multiplier:
return repeats
return int(math.ceil(multiplier * repeats))
def drop_connect(inputs, p, training):

 if not training: return inputs
batch_size = inputs.shape[0]
keep_prob = 1 - p
random_tensor = keep_prob
random_tensor += torch.rand([batch_size, 1, 1, 1], dtype=inputs.dtype, device=inputs.device)
 binary_tensor = torch.floor(random_tensor)
 output = inputs / keep_prob * binary_tensor
return output
def get_same_padding_conv2d(image_size=None):

 if image_size is None:
return Conv2dDynamicSamePadding
else:
return partial(Conv2dStaticSamePadding, image_size=image_size)
 class Conv2dDynamicSamePadding(nn.Conv2d):

 def __init__(self, in_channels, out_channels, kernel_size, stride=1, dilation=1, groups=1, bias=True):
super().__init__(in_channels, out_channels, kernel_size, stride, 0, dilation, groups, bias)
 self.stride = self.stride if len(self.stride)== 2 else [self.stride[0]]*2
def forward(self, x):
ih, iw = x.size() [-2:]
kh, kw = self.weight.size() [-2:]
sh, sw = self.stride
oh, ow = math.ceil(ih / sh), math.ceil(iw / sw)
 pad_h = max(( oh - 1)* self.stride[0] +(kh - 1)* self.dilation[0] + 1 - ih, 0)
 pad_w = max(( ow - 1)* self.stride[1] +(kw - 1)* self.dilation[1] + 1 - iw, 0)
 if pad_h > 0 or pad_w > 0:
x = F.pad(x, [pad_w//2, pad_w - pad_w//2, pad_h//2, pad_h - pad_h//2])
 return F.conv2d(x, self.weight, self.bias, self.stride, self.padding, self.dilation, self.groups)
 class Conv2dStaticSamePadding(nn.Conv2d):

 def __init__(self, in_channels, out_channels, kernel_size, image_size=None, **kwargs):
super().__init__(in_channels, out_channels, kernel_size, **kwargs)
 self.stride = self.stride if len(self.stride)== 2 else [self.stride[0]] * 2
assert image_size is not None
ih, iw = image_size if type(image_size)== list else [image_size, image_size]
kh, kw = self.weight.size() [-2:]
sh, sw = self.stride
oh, ow = math.ceil(ih / sh), math.ceil(iw / sw)
 pad_h = max(( oh - 1)* self.stride[0] +(kh - 1)* self.dilation[0] + 1 - ih, 0)
 pad_w = max(( ow - 1)* self.stride[1] +(kw - 1)* self.dilation[1] + 1 - iw, 0)
 if pad_h > 0 or pad_w > 0:
self.static_padding = nn.ZeroPad2d(( pad_w // 2, pad_w - pad_w // 2, pad_h // 2, pad_h - pad_h // 2))
else:
self.static_padding = Identity()
def forward(self, x):
x = self.static_padding(x)
 x = F.conv2d(x, self.weight, self.bias, self.stride, self.padding, self.dilation, self.groups)
 return x
class Identity(nn.Module):
def __init__(self,):
super(Identity, self ).__init__()
def forward(self, input):
return input
def efficientnet_params(model_name):

 params_dict = {
'efficientnet-b0':(1.0, 1.0, 224, 0.2),
'efficientnet-b1':(1.0, 1.1, 240, 0.2),
'efficientnet-b2':(1.1, 1.2, 260, 0.2),
'efficientnet-b3':(1.2, 1.4, 300, 0.2),
'efficientnet-b4':(1.4, 1.8, 380, 0.2),
'efficientnet-b5':(1.6, 2.2, 456, 0.2),
'efficientnet-b6':(1.8, 2.6, 528, 0.2),
'efficientnet-b7':(2.0, 3.1, 600, 0.2),
}
return params_dict[model_name]
class BlockDecoder(object):

 @staticmethod
def _decode_block_string(block_string):

 assert isinstance(block_string, str)
 ops = block_string.split('_')
 options = {}
for op in ops:
splits = re.split(r'(\d.*)', op)
 if len(splits)>= 2:
key, value = splits[:2]
options[key] = value
assert(( 's' in options and len(options['s'])== 1)or 
(len(options['s'])== 2 and options['s'][0] == options['s'][1]))
return BlockArgs(
 kernel_size=int(options['k']),
num_repeat=int(options['r']),
input_filters=int(options['i']),
output_filters=int(options['o']),
expand_ratio=int(options['e']),
id_skip=('noskip' not in block_string),
se_ratio=float(options['se'])if 'se' in options else None,
stride=[int(options['s'][0])])
 @staticmethod
def _encode_block_string(block):

 args = [
'r%d' % block.num_repeat,
'k%d' % block.kernel_size,
's%d%d' %(block.strides[0], block.strides[1]),
'e%s' % block.expand_ratio,
'i%d' % block.input_filters,
'o%d' % block.output_filters
]
if 0 < block.se_ratio <= 1:
args.append('se%s' % block.se_ratio)
 if block.id_skip is False:
args.append('noskip')
 return '_'.join(args)
 @staticmethod
def decode(string_list):

 assert isinstance(string_list, list)
 blocks_args = []
for block_string in string_list:
blocks_args.append(BlockDecoder._decode_block_string(block_string))
return blocks_args
@staticmethod
def encode(blocks_args):

 block_strings = []
for block in blocks_args:
block_strings.append(BlockDecoder._encode_block_string(block))
return block_strings
def efficientnet(width_coefficient=None, depth_coefficient=None, dropout_rate=0.2,
drop_connect_rate=0.2, image_size=None, num_classes=1000):

 blocks_args = [
'r1_k3_s11_e1_i32_o16_se0.25', 'r2_k3_s22_e6_i16_o24_se0.25',
'r2_k5_s22_e6_i24_o40_se0.25', 'r3_k3_s22_e6_i40_o80_se0.25',
'r3_k5_s11_e6_i80_o112_se0.25', 'r4_k5_s22_e6_i112_o192_se0.25',
'r1_k3_s11_e6_i192_o320_se0.25',
]
blocks_args = BlockDecoder.decode(blocks_args)
 global_params = GlobalParams(
 batch_norm_momentum=0.99,
batch_norm_epsilon=1e-4,
dropout_rate=dropout_rate,
drop_connect_rate=drop_connect_rate,
num_classes=num_classes,
width_coefficient=width_coefficient,
depth_coefficient=depth_coefficient,
depth_divisor=8,
min_depth=None,
image_size=image_size, 
)
 return blocks_args, global_params
def get_model_params(model_name, override_params):

 if model_name.startswith('efficientnet'):
w, d, s, p = efficientnet_params(model_name)
 blocks_args, global_params = efficientnet(
 width_coefficient=w, depth_coefficient=d, dropout_rate=p, image_size=s)
 else:
raise NotImplementedError('model name is not pre-defined: %s' % model_name)
 if override_params:
global_params = global_params._replace(**override_params)
 return blocks_args, global_params
url_map = {
'efficientnet-b0': 'http://storage.googleapis.com/public-models/efficientnet-b0-08094119.pth',
'efficientnet-b1': 'http://storage.googleapis.com/public-models/efficientnet-b1-dbc7070a.pth',
'efficientnet-b2': 'http://storage.googleapis.com/public-models/efficientnet-b2-27687264.pth',
'efficientnet-b3': 'http://storage.googleapis.com/public-models/efficientnet-b3-c8376fa2.pth',
'efficientnet-b4': 'http://storage.googleapis.com/public-models/efficientnet-b4-e116e8b3.pth',
'efficientnet-b5': 'http://storage.googleapis.com/public-models/efficientnet-b5-586e6cc6.pth',
}
def load_pretrained_weights(model, model_name, load_fc=True):

 state_dict = model_zoo.load_url(url_map[model_name])
 if load_fc:
model.load_state_dict(state_dict)
 else:
state_dict.pop('_fc.weight')
 state_dict.pop('_fc.bias')
 res = model.load_state_dict(state_dict, strict=False)
 assert str(res.missing_keys)== str(['_fc.weight', '_fc.bias']), 'issue loading pretrained weights'
print('Loaded pretrained weights for {}'.format(model_name))
class MBConvBlock(nn.Module):

 def __init__(self, block_args, global_params):
super().__init__()
self._block_args = block_args
self._bn_mom = 1 - global_params.batch_norm_momentum
self._bn_eps = global_params.batch_norm_epsilon
self.has_se =(self._block_args.se_ratio is not None)and(0 < self._block_args.se_ratio <= 1)
 self.id_skip = block_args.id_skip
Conv2d = get_same_padding_conv2d(image_size=global_params.image_size)
 inp = self._block_args.input_filters
oup = self._block_args.input_filters * self._block_args.expand_ratio
if self._block_args.expand_ratio != 1:
self._expand_conv = Conv2d(in_channels=inp, out_channels=oup, kernel_size=1, bias=False)
 self._bn0 = nn.BatchNorm2d(num_features=oup, momentum=self._bn_mom, eps=self._bn_eps)
 k = self._block_args.kernel_size
s = self._block_args.stride
self._depthwise_conv = Conv2d(
 in_channels=oup, out_channels=oup, groups=oup,
kernel_size=k, stride=s, bias=False)
 self._bn1 = nn.BatchNorm2d(num_features=oup, momentum=self._bn_mom, eps=self._bn_eps)
 if self.has_se:
num_squeezed_channels = max(1, int(self._block_args.input_filters * self._block_args.se_ratio))
self._se_reduce = Conv2d(in_channels=oup, out_channels=num_squeezed_channels, kernel_size=1)
 self._se_expand = Conv2d(in_channels=num_squeezed_channels, out_channels=oup, kernel_size=1)
 final_oup = self._block_args.output_filters
self._project_conv = Conv2d(in_channels=oup, out_channels=final_oup, kernel_size=1, bias=False)
 self._bn2 = nn.BatchNorm2d(num_features=final_oup, momentum=self._bn_mom, eps=self._bn_eps)
 def forward(self, inputs, drop_connect_rate=None):

 x = inputs
if self._block_args.expand_ratio != 1:
x = relu_fn(self._bn0(self._expand_conv(inputs)))
 x = relu_fn(self._bn1(self._depthwise_conv(x)))
 if self.has_se:
x_squeezed = F.adaptive_avg_pool2d(x, 1)
 x_squeezed = self._se_expand(relu_fn(self._se_reduce(x_squeezed)))
 x = torch.sigmoid(x_squeezed)* x
x = self._bn2(self._project_conv(x))
input_filters, output_filters = self._block_args.input_filters, self._block_args.output_filters
if self.id_skip and self._block_args.stride == 1 and input_filters == output_filters:
if drop_connect_rate:
x = drop_connect(x, p=drop_connect_rate, training=self.training)
 x = x + inputs
return x
class EfficientNet(nn.Module):

 def __init__(self, blocks_args=None, global_params=None):
super().__init__()
assert isinstance(blocks_args, list), 'blocks_args should be a list'
assert len(blocks_args)> 0, 'block args must be greater than 0'
self._global_params = global_params
self._blocks_args = blocks_args
Conv2d = get_same_padding_conv2d(image_size=global_params.image_size)
 bn_mom = 1 - self._global_params.batch_norm_momentum
bn_eps = self._global_params.batch_norm_epsilon
in_channels = 3
out_channels = round_filters(32, self._global_params)
 self._conv_stem = Conv2d(in_channels, out_channels, kernel_size=3, stride=2, bias=False)
 self._bn0 = nn.BatchNorm2d(num_features=out_channels, momentum=bn_mom, eps=bn_eps)
 self._blocks = nn.ModuleList([])
 for block_args in self._blocks_args:
block_args = block_args._replace(
 input_filters=round_filters(block_args.input_filters, self._global_params),
output_filters=round_filters(block_args.output_filters, self._global_params),
num_repeat=round_repeats(block_args.num_repeat, self._global_params)
)
 self._blocks.append(MBConvBlock(block_args, self._global_params))
if block_args.num_repeat > 1:
block_args = block_args._replace(input_filters=block_args.output_filters, stride=1)
 for _ in range(block_args.num_repeat - 1):
self._blocks.append(MBConvBlock(block_args, self._global_params))
in_channels = block_args.output_filters
out_channels = round_filters(1280, self._global_params)
 self._conv_head = Conv2d(in_channels, out_channels, kernel_size=1, bias=False)
 self._bn1 = nn.BatchNorm2d(num_features=out_channels, momentum=bn_mom, eps=bn_eps)
 self._dropout = self._global_params.dropout_rate
self._fc = nn.Linear(out_channels, self._global_params.num_classes)
 def extract_features(self, inputs):

 x = relu_fn(self._bn0(self._conv_stem(inputs)))
 for idx, block in enumerate(self._blocks):
drop_connect_rate = self._global_params.drop_connect_rate
if drop_connect_rate:
drop_connect_rate *= float(idx)/ len(self._blocks)
 x = block(x, drop_connect_rate=drop_connect_rate)
 x = relu_fn(self._bn1(self._conv_head(x)))
 return x
def forward(self, inputs):

 x = self.extract_features(inputs)
 x = F.adaptive_avg_pool2d(x, 1 ).squeeze(-1 ).squeeze(-1)
 if self._dropout:
x = F.dropout(x, p=self._dropout, training=self.training)
 x = self._fc(x)
 return x
@classmethod
def from_name(cls, model_name, override_params=None):
cls._check_model_name_is_valid(model_name)
 blocks_args, global_params = get_model_params(model_name, override_params)
 return EfficientNet(blocks_args, global_params)
 @classmethod
def from_pretrained(cls, model_name, num_classes=1000):
model = EfficientNet.from_name(model_name, override_params={'num_classes': num_classes})
 return model
@classmethod
def get_image_size(cls, model_name):
cls._check_model_name_is_valid(model_name)
 _, _, res, _ = efficientnet_params(model_name)
 return res
@classmethod
def _check_model_name_is_valid(cls, model_name, also_need_pretrained_weights=False):

 num_models = 4 if also_need_pretrained_weights else 8
valid_models = ['efficientnet_b'+str(i)for i in range(num_models)]
if model_name.replace('-','_')not in valid_models:
raise ValueError('model_name should be one of: ' + ', '.join(valid_models))
md_ef = EfficientNet.from_pretrained('efficientnet-b5', num_classes=1)
 !mkdir models
!cp '.. /input/kaggle-public-copy/abcdef.pth' 'models'
def get_df() :
base_image_dir = os.path.join('.. ', 'input/aptos2019-blindness-detection/')
 train_dir = os.path.join(base_image_dir,'train_images/')
 df = pd.read_csv(os.path.join(base_image_dir, 'train.csv'))
df['path'] = df['id_code'].map(lambda x: os.path.join(train_dir,'{}.png'.format(x)))
 df = df.drop(columns=['id_code'])
 df = df.sample(frac=1 ).reset_index(drop=True)
 test_df = pd.read_csv('.. /input/aptos2019-blindness-detection/sample_submission.csv')
 return df, test_df
df, test_df = get_df()
bs = 64
sz = 224
tfms = get_transforms(do_flip=True,flip_vert=True)
 data =(ImageList.from_df(df=df,path='./',cols='path')
.split_by_rand_pct(0.2)
.label_from_df(cols='diagnosis',label_cls=FloatList)
.transform(tfms,size=sz,resize_method=ResizeMethod.SQUISH,padding_mode='zeros')
.databunch(bs=bs,num_workers=4)
.normalize(imagenet_stats)
)
 def qk(y_pred, y):
return torch.tensor(cohen_kappa_score(torch.round(y_pred), y, weights='quadratic'), device='cuda:0')
 learn = Learner(data,
md_ef,
metrics = [qk],
model_dir="models" ).to_fp16()
learn.data.add_test(ImageList.from_df(test_df,
'.. /input/aptos2019-blindness-detection',
folder='test_images',
suffix='.png'))
learn.load('abcdef');
class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
ll = metrics.cohen_kappa_score(y, X_p, weights='quadratic')
 return -ll
def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X=X, y=y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 print(-loss_partial(self.coef_['x']))
def predict(self, X, coef):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
return X_p
def coefficients(self):
return self.coef_['x']
coefficients=[0.5, 1.5, 2.5, 3.5]
opt = OptimizedRounder()
preds,y = learn.get_preds(DatasetType.Test)
 preds4 = np.copy(preds)
 tst_pred = opt.predict(preds, coefficients)
 test_df.diagnosis = tst_pred.astype(int)
 submission4 = test_df.copy()
print('done' )<define_search_space>
pd.set_option('display.max_columns', 30 )
score = [0.777, 0.758, 0.783]
weight = [0.35, 0.2, 0.45]
predsData = weight[0]*preds1 + weight[1]*preds2 + weight[2]*preds4<save_to_csv>
at4_train =pd.read_csv(at4_train_path,sep=",")
 at2_train =pd.read_csv(at2_train_path,sep=",")
 bt_train=pd.read_csv(bt_train_path,sep=",")
 test_data =pd.read_csv(at4_test_path,sep="," )
coef = [0.5, 1.5, 2.5, 3.5]
for i, pred in enumerate(predsData):
if pred < coef[0]:
predsData[i] = 0
elif pred >= coef[0] and pred < coef[1]:
predsData[i] = 1
elif pred >= coef[1] and pred < coef[2]:
predsData[i] = 2
elif pred >= coef[2] and pred < coef[3]:
predsData[i] = 3
else:
predsData[i] = 4
submission = pd.read_csv(".. /input/aptos2019-blindness-detection/sample_submission.csv")
 submission.diagnosis = predsData.astype(int)
 submission.to_csv('submission.csv', index=False )<import_modules>
for i in range(1,31):
anova=calculate_anova(at2_train[at2_train['label']==3].iloc[:,i],
at4_train[at4_train['label']==3].iloc[:,i],
bt_train[bt_train['label']==3].iloc[:,i])
 print(i)
 print(anova )
__all__ = ['SENet', 'senet154', 'se_resnet50', 'se_resnet101', 'se_resnet152',
'se_resnext50_32x4d', 'se_resnext101_32x4d']
pretrained_settings = {
'senet154': {
'imagenet': {
'url': 'http://data.lip6.fr/cadene/pretrainedmodels/senet154-c7b49a05.pth',
'input_space': 'RGB',
'input_size': [3, 224, 224],
'input_range': [0, 1],
'mean': [0.485, 0.456, 0.406],
'std': [0.229, 0.224, 0.225],
'num_classes': 1000
}
},
'se_resnet50': {
'imagenet': {
'url': 'http://data.lip6.fr/cadene/pretrainedmodels/se_resnet50-ce0d4300.pth',
'input_space': 'RGB',
'input_size': [3, 224, 224],
'input_range': [0, 1],
'mean': [0.485, 0.456, 0.406],
'std': [0.229, 0.224, 0.225],
'num_classes': 1000
}
},
'se_resnet101': {
'imagenet': {
'url': 'http://data.lip6.fr/cadene/pretrainedmodels/se_resnet101-7e38fcc6.pth',
'input_space': 'RGB',
'input_size': [3, 224, 224],
'input_range': [0, 1],
'mean': [0.485, 0.456, 0.406],
'std': [0.229, 0.224, 0.225],
'num_classes': 1000
}
},
'se_resnet152': {
'imagenet': {
'url': 'http://data.lip6.fr/cadene/pretrainedmodels/se_resnet152-d17c99b7.pth',
'input_space': 'RGB',
'input_size': [3, 224, 224],
'input_range': [0, 1],
'mean': [0.485, 0.456, 0.406],
'std': [0.229, 0.224, 0.225],
'num_classes': 1000
}
},
'se_resnext50_32x4d': {
'imagenet': {
'url': 'http://data.lip6.fr/cadene/pretrainedmodels/se_resnext50_32x4d-a260b3a4.pth',
'input_space': 'RGB',
'input_size': [3, 224, 224],
'input_range': [0, 1],
'mean': [0.485, 0.456, 0.406],
'std': [0.229, 0.224, 0.225],
'num_classes': 1000
}
},
'se_resnext101_32x4d': {
'imagenet': {
'url': 'http://data.lip6.fr/cadene/pretrainedmodels/se_resnext101_32x4d-3b2fe3d8.pth',
'input_space': 'RGB',
'input_size': [3, 224, 224],
'input_range': [0, 1],
'mean': [0.485, 0.456, 0.406],
'std': [0.229, 0.224, 0.225],
'num_classes': 1000
}
},
}
class SEModule(nn.Module):
def __init__(self, channels, reduction):
super(SEModule, self ).__init__()
self.avg_pool = nn.AdaptiveAvgPool2d(1)
 self.fc1 = nn.Conv2d(channels, channels // reduction, kernel_size=1,
padding=0)
 self.relu = nn.ReLU(inplace=True)
 self.fc2 = nn.Conv2d(channels // reduction, channels, kernel_size=1,
padding=0)
 self.sigmoid = nn.Sigmoid()
def forward(self, x):
module_input = x
x = self.avg_pool(x)
 x = self.fc1(x)
 x = self.relu(x)
 x = self.fc2(x)
 x = self.sigmoid(x)
 return module_input * x
class Bottleneck(nn.Module):

 def forward(self, x):
residual = x
out = self.conv1(x)
 out = self.bn1(out)
 out = self.relu(out)
 out = self.conv2(out)
 out = self.bn2(out)
 out = self.relu(out)
 out = self.conv3(out)
 out = self.bn3(out)
 if self.downsample is not None:
residual = self.downsample(x)
 out = self.se_module(out)+ residual
out = self.relu(out)
 return out
class SEBottleneck(Bottleneck):

 expansion = 4
def __init__(self, inplanes, planes, groups, reduction, stride=1,
downsample=None):
super(SEBottleneck, self ).__init__()
self.conv1 = nn.Conv2d(inplanes, planes * 2, kernel_size=1, bias=False)
 self.bn1 = nn.BatchNorm2d(planes * 2)
 self.conv2 = nn.Conv2d(planes * 2, planes * 4, kernel_size=3,
stride=stride, padding=1, groups=groups,
bias=False)
 self.bn2 = nn.BatchNorm2d(planes * 4)
 self.conv3 = nn.Conv2d(planes * 4, planes * 4, kernel_size=1,
bias=False)
 self.bn3 = nn.BatchNorm2d(planes * 4)
 self.relu = nn.ReLU(inplace=True)
 self.se_module = SEModule(planes * 4, reduction=reduction)
 self.downsample = downsample
self.stride = stride
class SEResNetBottleneck(Bottleneck):

 expansion = 4
def __init__(self, inplanes, planes, groups, reduction, stride=1,
downsample=None):
super(SEResNetBottleneck, self ).__init__()
self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False,
stride=stride)
 self.bn1 = nn.BatchNorm2d(planes)
 self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, padding=1,
groups=groups, bias=False)
 self.bn2 = nn.BatchNorm2d(planes)
 self.conv3 = nn.Conv2d(planes, planes * 4, kernel_size=1, bias=False)
 self.bn3 = nn.BatchNorm2d(planes * 4)
 self.relu = nn.ReLU(inplace=True)
 self.se_module = SEModule(planes * 4, reduction=reduction)
 self.downsample = downsample
self.stride = stride
class SEResNeXtBottleneck(Bottleneck):

 expansion = 4
def __init__(self, inplanes, planes, groups, reduction, stride=1,
downsample=None, base_width=4):
super(SEResNeXtBottleneck, self ).__init__()
width = math.floor(planes *(base_width / 64)) * groups
self.conv1 = nn.Conv2d(inplanes, width, kernel_size=1, bias=False,
stride=1)
 self.bn1 = nn.BatchNorm2d(width)
 self.conv2 = nn.Conv2d(width, width, kernel_size=3, stride=stride,
padding=1, groups=groups, bias=False)
 self.bn2 = nn.BatchNorm2d(width)
 self.conv3 = nn.Conv2d(width, planes * 4, kernel_size=1, bias=False)
 self.bn3 = nn.BatchNorm2d(planes * 4)
 self.relu = nn.ReLU(inplace=True)
 self.se_module = SEModule(planes * 4, reduction=reduction)
 self.downsample = downsample
self.stride = stride
class SENet(nn.Module):
def __init__(self, block, layers, groups, reduction, dropout_p=0.2,
inplanes=128, input_3x3=True, downsample_kernel_size=3,
downsample_padding=1, num_classes=1000):

 super(SENet, self ).__init__()
self.inplanes = inplanes
if input_3x3:
layer0_modules = [ 
('conv1', nn.Conv2d(3, 64, 3, stride=2, padding=1,
bias=False)) , 
('bn1', nn.BatchNorm2d(64)) , 
('relu1', nn.ReLU(inplace=True)) , 
('conv2', nn.Conv2d(64, 64, 3, stride=1, padding=1,
bias=False)) , 
('bn2', nn.BatchNorm2d(64)) , 
('relu2', nn.ReLU(inplace=True)) , 
('conv3', nn.Conv2d(64, inplanes, 3, stride=1, padding=1,
bias=False)) , 
('bn3', nn.BatchNorm2d(inplanes)) , 
('relu3', nn.ReLU(inplace=True)) ,
]
else:
layer0_modules = [ 
('conv1', nn.Conv2d(3, inplanes, kernel_size=7, stride=2,
padding=3, bias=False)) , 
('bn1', nn.BatchNorm2d(inplanes)) , 
('relu1', nn.ReLU(inplace=True)) ,
]
layer0_modules.append(( 'pool', nn.MaxPool2d(3, stride=2,
ceil_mode=True)))
 self.layer0 = nn.Sequential(OrderedDict(layer0_modules))
self.layer1 = self._make_layer(
 block,
planes=64,
blocks=layers[0],
groups=groups,
reduction=reduction,
downsample_kernel_size=1,
downsample_padding=0 
)
 self.layer2 = self._make_layer(
 block,
planes=128,
blocks=layers[1],
stride=2,
groups=groups,
reduction=reduction,
downsample_kernel_size=downsample_kernel_size,
downsample_padding=downsample_padding 
)
 self.layer3 = self._make_layer(
 block,
planes=256,
blocks=layers[2],
stride=2,
groups=groups,
reduction=reduction,
downsample_kernel_size=downsample_kernel_size,
downsample_padding=downsample_padding 
)
 self.layer4 = self._make_layer(
 block,
planes=512,
blocks=layers[3],
stride=2,
groups=groups,
reduction=reduction,
downsample_kernel_size=downsample_kernel_size,
downsample_padding=downsample_padding 
)
 self.avg_pool = nn.AvgPool2d(7, stride=1)
 self.dropout = nn.Dropout(dropout_p)if dropout_p is not None else None
self.last_linear = nn.Linear(512 * block.expansion, num_classes)
 num_classes = 5
self.label_linear = nn.Linear(512 * block.expansion, num_classes)
 self.relu_ = nn.ReLU(inplace=True)
 self.emb = nn.Embedding(num_classes, num_classes)
 self.emb.weight = nn.Parameter(torch.eye(num_classes))
def _make_layer(self, block, planes, blocks, groups, reduction, stride=1,
downsample_kernel_size=1, downsample_padding=0):
downsample = None
if stride != 1 or self.inplanes != planes * block.expansion:
downsample = nn.Sequential(
 nn.Conv2d(self.inplanes, planes * block.expansion,
kernel_size=downsample_kernel_size, stride=stride,
padding=downsample_padding, bias=False),
nn.BatchNorm2d(planes * block.expansion), 
)
 layers = []
layers.append(block(self.inplanes, planes, groups, reduction, stride,
downsample))
self.inplanes = planes * block.expansion
for i in range(1, blocks):
layers.append(block(self.inplanes, planes, groups, reduction))
return nn.Sequential(*layers)
 def features(self, x):
x = self.layer0(x)
 x = self.layer1(x)
 x = self.layer2(x)
 x = self.layer3(x)
 x = self.layer4(x)
 return x
def logits(self, x):
x = self.avg_pool(x)
 if self.dropout is not None:
x = self.dropout(x)
 x = x.view(x.size(0), -1)
 x2 = x
x = self.last_linear(x)
 x2 = self.label_linear(x2.detach())
 return x, x2
def forward(self, x, targets=None):
x = self.features(x)
 x, x2 = self.logits(x)
 if targets is not None:
tar = self.emb(targets ).cuda()
else:
tar=None
return x, x2, tar, self.emb.weight
def initialize_pretrained_model(model, num_classes, settings):
assert num_classes == settings['num_classes'], \
'num_classes should be {}, but is {}'.format(
 settings['num_classes'], num_classes)
 pretrained_dict=model_zoo.load_url(settings['url'])
 model_dict = model.state_dict()
pretrained_dict = {k: v for k, v in pretrained_dict.items() if k in model_dict}
model_dict.update(pretrained_dict)
 model.load_state_dict(model_dict)
 model.input_space = settings['input_space']
model.input_size = settings['input_size']
model.input_range = settings['input_range']
model.mean = settings['mean']
model.std = settings['std']
def se_resnext50_32x4d(num_classes=1000, pretrained='imagenet'):
model = SENet(SEResNeXtBottleneck, [3, 4, 6, 3], groups=32, reduction=16,
dropout_p=None, inplanes=64, input_3x3=False,
downsample_kernel_size=1, downsample_padding=0,
num_classes=num_classes)
 return model<choose_model_class>
for i in range(1,31):
wilcoxon=calculate_wilcoxon(at2_train[at2_train['label']==2].iloc[:,i],
at4_train[at4_train['label']==2].iloc[:,i])
 stat, p_value = wilcoxon
print(i)
 if(p_value>0.05):
print('P-value larger than Alpha')
 print(wilcoxon )
def get_modified_model(model_name='se_resnext50_32x4d', num_outputs=None, pretrained=True,
freeze_bn=False, dropout_p=0, **kwargs):
model=se_resnext50_32x4d()
model.avg_pool = nn.AdaptiveAvgPool2d(1)
 in_features = model.last_linear.in_features
if dropout_p == 0:
model.last_linear = nn.Linear(in_features, num_outputs)
 else:
model.last_linear = nn.Sequential(
 nn.Dropout(p=dropout_p),
nn.Linear(in_features, num_outputs))
if freeze_bn:
for m in model.modules() :
if isinstance(m, nn.BatchNorm2d):
m.weight.requires_grad = False
m.bias.requires_grad = False
return model<normalization>
bt_train.groupby('label')['label'].count()
def scale_radius(src, img_size, padding=False):
x = src[src.shape[0] // 2,...].sum(axis=1)
 r =(x > x.mean() / 10 ).sum() // 2
yx = src.sum(axis=2)
 region_props = measure.regionprops(( yx > yx.mean() / 10 ).astype('uint8'))
yc, xc = np.round(region_props[0].centroid ).astype('int')
 x1 = max(xc - r, 0)
 x2 = min(xc + r, src.shape[1] - 1)
 y1 = max(yc - r, 0)
 y2 = min(yc + r, src.shape[0] - 1)
 dst = src[y1:y2, x1:x2]
dst = cv2.resize(dst, dsize=None, fx=img_size/(2*r), fy=img_size/(2*r))
if padding:
pad_x =(img_size - dst.shape[1])// 2
pad_y =(img_size - dst.shape[0])// 2
dst = np.pad(dst,(( pad_y, pad_y),(pad_x, pad_x),(0, 0)) , 'constant')
 return dst
class Dataset(torch.utils.data.Dataset):
def __init__(self, img_paths, labels, transform=None, img_size=288, save_img=True):
self.img_paths = img_paths
self.labels = labels
self.transform = transform
self.img_size = img_size
self.save_img = save_img
def __getitem__(self, index):
img_path, label = self.img_paths[index], self.labels[index]
if 'train' in img_path:
img = cv2.imread('.. /input/aptos2019-resized/images_288_scaled/%s' %os.path.basename(img_path))
else:
if os.path.exists('processed/%s' %os.path.basename(img_path)) :
img = cv2.imread('processed/%s' %os.path.basename(img_path))
else:
img = cv2.imread(img_path)
 try:
img = scale_radius(img, img_size=self.img_size, padding=False)
 except Exception as e:
img = img
if self.save_img:
cv2.imwrite('processed/%s' %os.path.basename(img_path), img)
 img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
 img = Image.fromarray(img)
 if self.transform is not None:
img = self.transform(img)
 return img, label
def __len__(self):
return len(self.img_paths )<choose_model_class>
for i in range(10,21):
for j in range(i,21):
if(i !=j):
stat, p_value=calculate_wilcoxon(bt_train.iloc[:,i],bt_train.iloc[:,j])
 if(p_value>0.05 and i!=j):
print('columns {} and {} are similar'.format(i,j))
class RAdam(Optimizer):
def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8, weight_decay=0):
defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay)
 self.buffer = [[None, None, None] for ind in range(10)]
super(RAdam, self ).__init__(params, defaults)
 def __setstate__(self, state):
super(RAdam, self ).__setstate__(state)
 def step(self, closure=None):
loss = None
if closure is not None:
loss = closure()
for group in self.param_groups:
for p in group['params']:
if p.grad is None:
continue
grad = p.grad.data.float()
if grad.is_sparse:
raise RuntimeError('RAdam does not support sparse gradients')
 p_data_fp32 = p.data.float()
state = self.state[p]
if len(state)== 0:
state['step'] = 0
state['exp_avg'] = torch.zeros_like(p_data_fp32)
 state['exp_avg_sq'] = torch.zeros_like(p_data_fp32)
 else:
state['exp_avg'] = state['exp_avg'].type_as(p_data_fp32)
 state['exp_avg_sq'] = state['exp_avg_sq'].type_as(p_data_fp32)
 exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq']
beta1, beta2 = group['betas']
exp_avg_sq.mul_(beta2 ).addcmul_(1 - beta2, grad, grad)
 exp_avg.mul_(beta1 ).add_(1 - beta1, grad)
 state['step'] += 1
buffered = self.buffer[int(state['step'] % 10)]
if state['step'] == buffered[0]:
N_sma, step_size = buffered[1], buffered[2]
else:
buffered[0] = state['step']
beta2_t = beta2 ** state['step']
N_sma_max = 2 /(1 - beta2)- 1
N_sma = N_sma_max - 2 * state['step'] * beta2_t /(1 - beta2_t)
 buffered[1] = N_sma
if N_sma >= 5:
step_size = group['lr'] * math.sqrt(( 1 - beta2_t)*(N_sma - 4)/(N_sma_max - 4)*(N_sma - 2)/ N_sma * N_sma_max /(N_sma_max - 2)) /(1 - beta1 ** state['step'])
 else:
step_size = group['lr'] /(1 - beta1 ** state['step'])
 buffered[2] = step_size
if group['weight_decay'] != 0:
p_data_fp32.add_(-group['weight_decay'] * group['lr'], p_data_fp32)
 if N_sma >= 5:
denom = exp_avg_sq.sqrt().add_(group['eps'])
 p_data_fp32.addcdiv_(-step_size, exp_avg, denom)
 else:
p_data_fp32.add_(-step_size, exp_avg)
 p.data.copy_(p_data_fp32)
 return loss
class AverageMeter(object):

 def __init__(self):
self.reset()
def reset(self):
self.val = 0
self.avg = 0
self.sum = 0
self.count = 0
def update(self, val, n=1):
self.val = val
self.sum += val * n
self.count += n
self.avg = self.sum / self.count
def quadratic_weighted_kappa(y_pred, y_true):
if torch.is_tensor(y_pred):
y_pred = y_pred.data.cpu().numpy()
if torch.is_tensor(y_true):
y_true = y_true.data.cpu().numpy()
if y_pred.shape[1] == 1:
y_pred = y_pred[:, 0]
else:
y_pred = np.argmax(y_pred, axis=1)
 return metrics.cohen_kappa_score(y_pred, y_true, weights='quadratic')
 def comp_Loss(out1, out2, tar, emb_w, targets):

 out2_prob = F.softmax(out2, dim=1)
 tau2_prob = F.softmax(out2 / 2., dim=1 ).detach()
soft_tar = F.softmax(tar, dim=1 ).detach()
L_o1_y = F.cross_entropy(out1, targets)
 alpha = 0.9
beta = 0.5
_, pred = torch.max(out2,1)
 mask = pred.eq(targets ).float().detach()
L_o1_emb = -torch.mean(my_loss(out1, soft_tar))
L_o2_y = F.cross_entropy(out2, targets)
 L_emb_o2 = -torch.sum(my_loss(tar, tau2_prob)*mask)/(torch.sum(mask)+1e-8)
 gap = torch.gather(out2_prob, 1, targets.view(-1,1)) -alpha
L_re = torch.sum(F.relu(gap))
loss = beta*L_o1_y +(1-beta)*L_o1_emb +L_o2_y +L_emb_o2 +L_re
return loss
def my_loss(logit, prob):

 soft_logit = F.log_softmax(logit, dim=1)
 loss = torch.sum(prob*soft_logit, 1)
 return loss
def compute_accuracy(y_pred,y_true):
if torch.is_tensor(y_pred):
y_pred = y_pred.data.cpu().numpy()
if torch.is_tensor(y_true):
y_true = y_true.data.cpu().numpy()
if y_pred.shape[1] == 1:
y_pred = y_pred[:, 0]
else:
y_pred = np.argmax(y_pred, axis=1)
 return metrics.accuracy_score(y_true, y_pred, normalize=True )<train_on_grid>
for i in range(20,31):
for j in range(i,31):
if(i !=j):
stat, p_value=calculate_wilcoxon(bt_train.iloc[:,i],bt_train.iloc[:,j])
 if(p_value>0.05 and i!=j):
print('columns {} and {} are similar'.format(i,j))
def my_train(train_loader, model, criterion, optimizer, epoch):
losses = AverageMeter()
scores = AverageMeter()
ac_scores = AverageMeter()
model.train()
for i,(input, target)in tqdm(enumerate(train_loader), total=len(train_loader)) :
input = input.cuda()
target = target.cuda()
output = model(input, target)
 out1, out2, tar, emb_w = output
loss = comp_Loss(out1, out2, tar, emb_w, target)
 optimizer.zero_grad()
loss.backward()
optimizer.step()
score = quadratic_weighted_kappa(out1, target)
 ac_score = compute_accuracy(out1, target)
 losses.update(loss.item() , input.size(0))
scores.update(score, input.size(0))
ac_scores.update(ac_score, input.size(0))
return losses.avg, scores.avg, ac_scores.avg
def my_validate(val_loader, model, criterion):
losses = AverageMeter()
scores = AverageMeter()
ac_scores = AverageMeter()
model.eval()
with torch.no_grad() :
for i,(input, target)in tqdm(enumerate(val_loader), total=len(val_loader)) :
input = input.cuda()
target = target.cuda()
output = model(input, target)
 out1, out2, tar, emb_w = output
loss = comp_Loss(out1, out2, tar, emb_w, target)
 score = quadratic_weighted_kappa(out1, target)
 ac_score = compute_accuracy(out1, target)
 losses.update(loss.item() , input.size(0))
scores.update(score, input.size(0))
ac_scores.update(ac_score, input.size(0))
return losses.avg, scores.avg, ac_scores.avg<load_from_csv>
at4_train.loc[:,at4_train.columns!='label'].apply(minMax )
pseudo_probs = {}
aptos2019_df = pd.read_csv('.. /input/aptos2019-blindness-detection/train.csv')
 aptos2019_img_paths = '.. /input/aptos2019-blindness-detection/train_images/' + aptos2019_df['id_code'].values + '.png'
aptos2019_labels = aptos2019_df['diagnosis'].values
test_df = pd.read_csv('.. /input/aptos2019-blindness-detection/test.csv')
 dir_name = '.. /input/aptos2019-blindness-detection/test_images'
test_img_paths = dir_name + '/' + test_df['id_code'].values + '.png'
test_labels = np.zeros(len(test_img_paths))
test_transform = transforms.Compose([
transforms.Resize(( 256, 256)) ,
transforms.ToTensor() ,
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),
])
 test_set = Dataset(
 test_img_paths,
test_labels,
transform=test_transform)
 test_loader = torch.utils.data.DataLoader(
 test_set,
batch_size=32,
shuffle=False,
num_workers=2 )<load_pretrained>
at4_train.loc[:,at4_train.columns!='label'].apply(minMax )
model = get_modified_model(model_name='se_resnext50_32x4d',
num_outputs=5,
pretrained=False,
freeze_bn=True,
dropout_p=0)
 model = model.cuda()
model.eval()
probs = []
for fold in range(5):
print('Fold [%d/%d]' %(fold+1, 5))
model.load_state_dict(torch.load('.. /input/my-se-resnext50-32x4d-10211607/model_1_%d.pth' %(fold+1)))
 probs_fold = []
with torch.no_grad() :
for i,(input, _)in tqdm(enumerate(test_loader), total=len(test_loader)) :
input = input.cuda()
output,_,_,_ = model(input)
 probs_fold.extend(output.data.cpu().numpy())
 probs_fold = np.array(probs_fold)
 probs.append(probs_fold)
 del model
torch.cuda.empty_cache()
!nvidia-smi<prepare_output>
bt_train.loc[:,bt_train.columns!='label'].apply(minMax )
result = np.mean(probs,axis=0)
 result = np.argmax(result,axis=1)
 pseudo_probs['se_resnext50_32x4d'] = result<load_from_csv>
X_At4_train, X_At4_val, y_At4_train, y_At4_val=train_test_split(at4_train.loc[: , at4_train.columns!='label'],at4_train['label'],random_state=0)
 X_At2_train, X_At2_val, y_At2_train, y_At2_val=train_test_split(at2_train.loc[: , at2_train.columns!='label'],at2_train['label'],random_state=0)
 X_bt_train, X_bt_val, y_bt_train, y_bt_val=train_test_split(bt_train.loc[: , bt_train.columns!='label'],bt_train['label'],random_state=0 )
aptos2019_df = pd.read_csv('.. /input/aptos2019-blindness-detection/train.csv')
 aptos2019_img_paths = '.. /input/aptos2019-blindness-detection/train_images/' + aptos2019_df['id_code'].values + '.png'
aptos2019_labels = aptos2019_df['diagnosis'].values
test_df = pd.read_csv('.. /input/aptos2019-blindness-detection/test.csv')
 test_img_paths = '.. /input/aptos2019-blindness-detection/test_images/' + test_df['id_code'].values + '.png'
test_labels = pseudo_probs['se_resnext50_32x4d']
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=41)
 kf = KFold(n_splits=5, shuffle=True, random_state=41)
 img_paths = []
labels = []
for(train_idx1, val_idx1),(train_idx2, val_idx2)in zip(skf.split(aptos2019_img_paths, aptos2019_labels), kf.split(test_img_paths)) :
img_paths.append(( np.hstack(( aptos2019_img_paths[train_idx1], test_img_paths[val_idx2])) , aptos2019_img_paths[val_idx1]))
labels.append(( np.hstack(( aptos2019_labels[train_idx1], test_labels[val_idx2])) , aptos2019_labels[val_idx1]))<choose_model_class>
def create_simple_mlp() :
mlp = Sequential()
mlp.add(Dense(32, activation='relu'))
mlp.add(Dense(16, activation='relu'))
mlp.add(Dense(8, activation='relu'))
mlp.add(Dense(4, activation='softmax'))
return mlp
model = get_modified_model(model_name='se_resnext50_32x4d',
num_outputs=5,
pretrained=False,
freeze_bn=True,
dropout_p=0, 
)
 model = model.cuda()
criterion = nn.CrossEntropyLoss().cuda()
best_losses = []
best_scores = []
for fold,(( train_img_paths, val_img_paths),(train_labels, val_labels)) in enumerate(zip(img_paths, labels)) :
print('Fold [%d/%d]' %(fold+1, len(img_paths)))
 train_set = Dataset(
 train_img_paths,
train_labels,
transform=train_transform,
img_size=288,
save_img=True)
 train_loader = torch.utils.data.DataLoader(
 train_set,
batch_size=32,
shuffle=True,
num_workers=2)
 val_set = Dataset(
 val_img_paths,
val_labels,
transform=val_transform,
save_img=True)
 val_loader = torch.utils.data.DataLoader(
 val_set,
batch_size=32,
shuffle=False,
num_workers=2)
 model.load_state_dict(torch.load('.. /input/my-se-resnext50-32x4d-10211607/model_1_%d.pth' %(fold+1)))
 optimizer = RAdam(
 filter(lambda p: p.requires_grad, model.parameters()), lr=1e-3)
 scheduler = lr_scheduler.CosineAnnealingLR(
 optimizer, T_max=10, eta_min=1e-5)
 os.makedirs('se_resnext50_32x4d', exist_ok=True)
 best_loss = float('inf')
 best_score = 0
best_ac_score = 0
for epoch in range(10):
print('Epoch [%d/%d]' %(epoch + 1, 10))
train_loss, train_score, train_ac_score = my_train(train_loader, model, criterion, optimizer, epoch)
 val_loss, val_score, val_ac_score = my_validate(val_loader, model, criterion)
 scheduler.step()
print('loss %.4f - score %.4f - ac_score %.4f - val_loss %.4f - val_score %.4f - val_ac_score %.4f'
%(train_loss, train_score, train_ac_score, val_loss, val_score, val_ac_score))
if val_loss < best_loss:
torch.save(model.state_dict() , 'se_resnext50_32x4d/model_%d.pth' %(fold+1))
best_loss = val_loss
best_score = val_score
best_ac_socre = val_ac_score
print("=> saved best model")
 print('val_loss: %f' % best_loss)
 print('val_score: %f' % best_score)
 print('val_ac_score: %f' % best_ac_score)
 del model
torch.cuda.empty_cache()
!nvidia-smi<load_from_csv>
opt = keras.optimizers.Adam(learning_rate=0.001)
 es = EarlyStopping(monitor='val_loss', mode='min', verbose=1,patience=50 )
test_df = pd.read_csv('.. /input/aptos2019-blindness-detection/test.csv')
 dir_name = '.. /input/aptos2019-blindness-detection/test_images'
test_img_paths = dir_name + '/' + test_df['id_code'].values + '.png'
test_labels = np.zeros(len(test_img_paths))
test_transform = transforms.Compose([
transforms.Resize(( 256, 256)) ,
transforms.ToTensor() ,
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),
])
 test_set = Dataset(
 test_img_paths,
test_labels,
transform=test_transform)
 test_loader = torch.utils.data.DataLoader(
 test_set,
batch_size=32,
shuffle=False,
num_workers=2)
 model = get_modified_model(model_name='se_resnext50_32x4d',
num_outputs=5,
pretrained=False,
freeze_bn=True,
dropout_p=0)
 model = model.cuda()
model.eval()
probs = []
for fold in range(5):
print('Fold [%d/%d]' %(fold+1, 5))
model.load_state_dict(torch.load('se_resnext50_32x4d/model_%d.pth' %(fold+1)))
 probs_fold = []
with torch.no_grad() :
for i,(input, _)in tqdm(enumerate(test_loader), total=len(test_loader)) :
input = input.cuda()
output,_,_,_ = model(input)
 probs_fold.extend(output.data.cpu().numpy())
 probs_fold = np.array(probs_fold)
 probs.append(probs_fold)
 probs = np.mean(probs, axis=0)
 probs = np.argmax(probs, axis=1)
 l1_probs['se_resnext50_32x4d'] = probs
del model
torch.cuda.empty_cache()
!nvidia-smi<save_to_csv>
mlp_bt=create_simple_mlp()
mlp_bt.compile(loss='sparse_categorical_crossentropy', optimizer=opt, metrics=['acc'])
 history_bt = mlp_bt.fit(X_bt_train,y_bt_train ,batch_size=32, validation_data=(X_bt_val, y_bt_val), epochs=400, callbacks=[es],verbose=0)
 plot_accuracy_and_loss(history_bt)
preds = l1_probs['se_resnext50_32x4d']
preds = preds.astype('int')
 test_df['diagnosis'] = preds
test_df.to_csv('submission.csv', index=False )<import_modules>
score_bt=mlp_bt.evaluate(X_bt_val,y_bt_val)
 score_bt
import numpy as np
import pandas as pd
from sklearn import metrics<save_to_csv>
mlp_at2=create_simple_mlp()
mlp_at2.compile(loss='sparse_categorical_crossentropy', optimizer=opt, metrics=['acc'])
 history_at2 = mlp_at2.fit(X_At2_train,y_At2_train ,batch_size=16, validation_data=(X_At4_val, y_At4_val), epochs=400, callbacks=[es],verbose=0)
 plot_accuracy_and_loss(history_at2)
df_sub = pd.read_csv(".. /input/aptos2019-blindness-detection/sample_submission.csv")
 df_tmp = pd.read_csv(".. /input/dr-submission/df_test_DR_Ensemble_ver5.csv")
 df_sub = pd.merge(df_sub[['id_code']], df_tmp[['id_code', 'pred_c']], on='id_code', how='left' ).fillna(0)
 df_sub.columns = ['id_code', 'diagnosis']
df_sub["diagnosis"] = df_sub["diagnosis"].astype(int)
 df_sub.to_csv("submission.csv", index=None)
 df_sub.head(30 )<set_options>
mlp_at4=create_simple_mlp()
mlp_at4.compile(loss='sparse_categorical_crossentropy', optimizer=opt, metrics=['acc'])
 history_at4 = mlp_at4.fit(X_At4_train,y_At4_train ,batch_size=16, validation_data=(X_At4_val, y_At4_val), epochs=400, callbacks=[es],verbose=0)
 plot_accuracy_and_loss(history_at4)
warnings.filterwarnings('ignore' )<import_modules>
shuffeled=X_At4_val.copy()
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.utils.model_zoo as model_zoo
import re
import math
from copy import deepcopy<load_pretrained>
def shuffle_and_test(i,j,df):
shuffeled=df.copy()
for i in range(i,j):
shuffeled.iloc[:,i]=np.random.permutation(shuffeled.iloc[:,i].values)
 print(mlp_at4.evaluate(shuffeled, y_At4_val))
def load_pretrained(model, default_cfg, num_classes=1000, in_chans=3, filter_fn=None):
if 'url' not in default_cfg or not default_cfg['url']:
logging.warning("Pretrained model URL is invalid, using random initialization.")
 return
state_dict = model_zoo.load_url(default_cfg['url'])
 if in_chans == 1:
conv1_name = default_cfg['first_conv']
logging.info('Converting first conv(%s)from 3 to 1 channel' % conv1_name)
 conv1_weight = state_dict[conv1_name + '.weight']
state_dict[conv1_name + '.weight'] = conv1_weight.sum(dim=1, keepdim=True)
 elif in_chans != 3:
assert False, "Invalid in_chans for pretrained weights"
strict = True
classifier_name = default_cfg['classifier']
if num_classes == 1000 and default_cfg['num_classes'] == 1001:
classifier_weight = state_dict[classifier_name + '.weight']
state_dict[classifier_name + '.weight'] = classifier_weight[1:]
classifier_bias = state_dict[classifier_name + '.bias']
state_dict[classifier_name + '.bias'] = classifier_bias[1:]
elif num_classes != default_cfg['num_classes']:
del state_dict[classifier_name + '.weight']
del state_dict[classifier_name + '.bias']
strict = False
if filter_fn is not None:
state_dict = filter_fn(state_dict)
 model.load_state_dict(state_dict, strict=strict)
<statistical_test>
shuffle_and_test(0,10,X_At4_val)
def adaptive_pool_feat_mult(pool_type='avg'):
if pool_type == 'catavgmax':
return 2
else:
return 1
def adaptive_avgmax_pool2d(x, output_size=1):
x_avg = F.adaptive_avg_pool2d(x, output_size)
 x_max = F.adaptive_max_pool2d(x, output_size)
 return 0.5 *(x_avg + x_max)
 def adaptive_catavgmax_pool2d(x, output_size=1):
x_avg = F.adaptive_avg_pool2d(x, output_size)
 x_max = F.adaptive_max_pool2d(x, output_size)
 return torch.cat(( x_avg, x_max), 1)
 def select_adaptive_pool2d(x, pool_type='avg', output_size=1):

 if pool_type == 'avg':
x = F.adaptive_avg_pool2d(x, output_size)
 elif pool_type == 'avgmax':
x = adaptive_avgmax_pool2d(x, output_size)
 elif pool_type == 'catavgmax':
x = adaptive_catavgmax_pool2d(x, output_size)
 elif pool_type == 'max':
x = F.adaptive_max_pool2d(x, output_size)
 else:
assert False, 'Invalid pool type: %s' % pool_type
return x
class AdaptiveAvgMaxPool2d(nn.Module):
def __init__(self, output_size=1):
super(AdaptiveAvgMaxPool2d, self ).__init__()
self.output_size = output_size
def forward(self, x):
return adaptive_avgmax_pool2d(x, self.output_size)
 class AdaptiveCatAvgMaxPool2d(nn.Module):
def __init__(self, output_size=1):
super(AdaptiveCatAvgMaxPool2d, self ).__init__()
self.output_size = output_size
def forward(self, x):
return adaptive_catavgmax_pool2d(x, self.output_size)
 class SelectAdaptivePool2d(nn.Module):

 def __init__(self, output_size=1, pool_type='avg'):
super(SelectAdaptivePool2d, self ).__init__()
self.output_size = output_size
self.pool_type = pool_type
if pool_type == 'avgmax':
self.pool = AdaptiveAvgMaxPool2d(output_size)
 elif pool_type == 'catavgmax':
self.pool = AdaptiveCatAvgMaxPool2d(output_size)
 elif pool_type == 'max':
self.pool = nn.AdaptiveMaxPool2d(output_size)
 else:
if pool_type != 'avg':
assert False, 'Invalid pool type: %s' % pool_type
self.pool = nn.AdaptiveAvgPool2d(output_size)
 def forward(self, x):
return self.pool(x)
 def feat_mult(self):
return adaptive_pool_feat_mult(self.pool_type)
 def __repr__(self):
return self.__class__.__name__ + '(' \
+ 'output_size=' + str(self.output_size)\
+ ', pool_type=' + self.pool_type + ')' 
<define_search_space>
shuffle_and_test(10,20,X_At4_val )
IMAGENET_DEFAULT_MEAN =(0.485, 0.456, 0.406)
 IMAGENET_DEFAULT_STD =(0.229, 0.224, 0.225)
 def _cfg(url='', **kwargs):
return {
'url': url, 'num_classes': 1000, 'input_size':(3, 224, 224), 'pool_size':(7, 7),
'crop_pct': 0.875, 'interpolation': 'bicubic',
'mean': IMAGENET_DEFAULT_MEAN, 'std': IMAGENET_DEFAULT_STD,
'first_conv': 'conv_stem', 'classifier': 'classifier',
**kwargs
}
def _is_static_pad(kernel_size, stride=1, dilation=1, **_):
return stride == 1 and(dilation *(kernel_size - 1)) % 2 == 0
def _get_padding(kernel_size, stride=1, dilation=1, **_):
padding =(( stride - 1)+ dilation *(kernel_size - 1)) // 2
return padding
def _calc_same_pad(i, k, s, d):
return max(( math.ceil(i / s)- 1)* s +(k - 1)* d + 1 - i, 0)
 def _split_channels(num_chan, num_groups):
split = [num_chan // num_groups for _ in range(num_groups)]
split[0] += num_chan - sum(split)
 return split
class Conv2dSame(nn.Conv2d):

 def __init__(self, in_channels, out_channels, kernel_size, stride=1,
padding=0, dilation=1, groups=1, bias=True):
super(Conv2dSame, self ).__init__(
 in_channels, out_channels, kernel_size, stride, 0, dilation,
groups, bias)
 def forward(self, x):
ih, iw = x.size() [-2:]
kh, kw = self.weight.size() [-2:]
pad_h = _calc_same_pad(ih, kh, self.stride[0], self.dilation[0])
 pad_w = _calc_same_pad(iw, kw, self.stride[1], self.dilation[1])
 if pad_h > 0 or pad_w > 0:
x = F.pad(x, [pad_w//2, pad_w - pad_w//2, pad_h//2, pad_h - pad_h//2])
 return F.conv2d(x, self.weight, self.bias, self.stride,
self.padding, self.dilation, self.groups)
 def conv2d_pad(in_chs, out_chs, kernel_size, **kwargs):
padding = kwargs.pop('padding', '')
 kwargs.setdefault('bias', False)
 if isinstance(padding, str):
padding = padding.lower()
if padding == 'same':
if _is_static_pad(kernel_size, **kwargs):
padding = _get_padding(kernel_size, **kwargs)
 return nn.Conv2d(in_chs, out_chs, kernel_size, padding=padding, **kwargs)
 else:
return Conv2dSame(in_chs, out_chs, kernel_size, **kwargs)
 elif padding == 'valid':
return nn.Conv2d(in_chs, out_chs, kernel_size, padding=0, **kwargs)
 else:
padding = _get_padding(kernel_size, **kwargs)
 return nn.Conv2d(in_chs, out_chs, kernel_size, padding=padding, **kwargs)
 else:
return nn.Conv2d(in_chs, out_chs, kernel_size, padding=padding, **kwargs)
 def select_conv2d(in_chs, out_chs, kernel_size, **kwargs):
assert 'groups' not in kwargs
if isinstance(kernel_size, list):
return MixedConv2d(in_chs, out_chs, kernel_size, **kwargs)
 else:
depthwise = kwargs.pop('depthwise', False)
 groups = out_chs if depthwise else 1
return conv2d_pad(in_chs, out_chs, kernel_size, groups=groups, **kwargs)
 def _parse_ksize(ss):
if ss.isdigit() :
return int(ss)
 else:
return [int(k)for k in ss.split('.')]
def _resolve_bn_args(kwargs):
bn_args = _BN_ARGS_TF.copy() if kwargs.pop('bn_tf', False)else _BN_ARGS_PT.copy()
bn_momentum = kwargs.pop('bn_momentum', None)
 if bn_momentum is not None:
bn_args['momentum'] = bn_momentum
bn_eps = kwargs.pop('bn_eps', None)
 if bn_eps is not None:
bn_args['eps'] = bn_eps
return bn_args
def _round_channels(channels, multiplier=1.0, divisor=8, channel_min=None):

 if not multiplier:
return channels
channels *= multiplier
channel_min = channel_min or divisor
new_channels = max(
 int(channels + divisor / 2)// divisor * divisor,
channel_min)
 if new_channels < 0.9 * channels:
new_channels += divisor
return new_channels
def _decode_arch_def(arch_def, depth_multiplier=1.0):
arch_args = []
for stack_idx, block_strings in enumerate(arch_def):
assert isinstance(block_strings, list)
 stack_args = []
for block_str in block_strings:
assert isinstance(block_str, str)
 stack_args.extend(_decode_block_str(block_str, depth_multiplier))
arch_args.append(stack_args)
 return arch_args
def _decode_block_str(block_str, depth_multiplier=1.0):

 assert isinstance(block_str, str)
 ops = block_str.split('_')
 block_type = ops[0]
ops = ops[1:]
options = {}
noskip = False
for op in ops:
if op == 'noskip':
noskip = True
elif op.startswith('n'):
key = op[0]
v = op[1:]
if v == 're':
value = F.relu
elif v == 'r6':
value = F.relu6
elif v == 'hs':
value = hard_swish
elif v == 'sw':
value = swish
else:
continue
options[key] = value
else:
splits = re.split(r'(\d.*)', op)
 if len(splits)>= 2:
key, value = splits[:2]
options[key] = value
act_fn = options['n'] if 'n' in options else None
exp_kernel_size = _parse_ksize(options['a'])if 'a' in options else 1
pw_kernel_size = _parse_ksize(options['p'])if 'p' in options else 1
num_repeat = int(options['r'])
 if block_type == 'ir':
block_args = dict(
 block_type=block_type,
dw_kernel_size=_parse_ksize(options['k']),
exp_kernel_size=exp_kernel_size,
pw_kernel_size=pw_kernel_size,
out_chs=int(options['c']),
exp_ratio=float(options['e']),
se_ratio=float(options['se'])if 'se' in options else None,
stride=int(options['s']),
act_fn=act_fn,
noskip=noskip, 
)
 elif block_type == 'ds' or block_type == 'dsa':
block_args = dict(
 block_type=block_type,
dw_kernel_size=_parse_ksize(options['k']),
pw_kernel_size=pw_kernel_size,
out_chs=int(options['c']),
se_ratio=float(options['se'])if 'se' in options else None,
stride=int(options['s']),
act_fn=act_fn,
pw_act=block_type == 'dsa',
noskip=block_type == 'dsa' or noskip, 
)
 elif block_type == 'cn':
block_args = dict(
 block_type=block_type,
kernel_size=int(options['k']),
out_chs=int(options['c']),
stride=int(options['s']),
act_fn=act_fn, 
)
 else:
assert False, 'Unknown block type(%s)' % block_type
num_repeat = int(math.ceil(num_repeat * depth_multiplier))
return [deepcopy(block_args)for _ in range(num_repeat)]
default_cfgs = {
'efficientnet_b0': _cfg(
 url='https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/efficientnet_b0-d6904d92.pth'),
'efficientnet_b1': _cfg(
 url='https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/efficientnet_b1-533bc792.pth',
input_size=(3, 240, 240), pool_size=(8, 8), crop_pct=0.882),
'efficientnet_b2': _cfg(
 url='https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/efficientnet_b2-cf78dc4d.pth',
input_size=(3, 260, 260), pool_size=(9, 9), crop_pct=0.890),
'efficientnet_b3': _cfg(
 url='', input_size=(3, 300, 300), pool_size=(10, 10), crop_pct=0.904),
'efficientnet_b4': _cfg(
 url='', input_size=(3, 380, 380), pool_size=(12, 12), crop_pct=0.922),
'efficientnet_b5': _cfg(
 url='', input_size=(3, 456, 456), pool_size=(15, 15), crop_pct=0.934),
'efficientnet_b6': _cfg(
 url='', input_size=(3, 528, 528), pool_size=(17, 17), crop_pct=0.942),
'efficientnet_b7': _cfg(
 url='', input_size=(3, 600, 600), pool_size=(19, 19), crop_pct=0.949),
'tf_efficientnet_b0': _cfg(
 url='https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/tf_efficientnet_b0_aa-827b6e33.pth',
input_size=(3, 224, 224)) ,
'tf_efficientnet_b1': _cfg(
 url='https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/tf_efficientnet_b1_aa-ea7a6ee0.pth',
input_size=(3, 240, 240), pool_size=(8, 8), crop_pct=0.882),
'tf_efficientnet_b2': _cfg(
 url='https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/tf_efficientnet_b2_aa-60c94f97.pth',
input_size=(3, 260, 260), pool_size=(9, 9), crop_pct=0.890),
'tf_efficientnet_b3': _cfg(
 url='https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/tf_efficientnet_b3_aa-84b4657e.pth',
input_size=(3, 300, 300), pool_size=(10, 10), crop_pct=0.904),
'tf_efficientnet_b4': _cfg(
 url='https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/tf_efficientnet_b4_aa-818f208c.pth',
input_size=(3, 380, 380), pool_size=(12, 12), crop_pct=0.922),
'tf_efficientnet_b5': _cfg(
 url='https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/tf_efficientnet_b5_aa-99018a74.pth',
input_size=(3, 456, 456), pool_size=(15, 15), crop_pct=0.934),
'tf_efficientnet_b6': _cfg(
 url='https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/tf_efficientnet_b6_aa-80ba17e4.pth',
input_size=(3, 528, 528), pool_size=(17, 17), crop_pct=0.942),
'tf_efficientnet_b7': _cfg(
 url='https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/tf_efficientnet_b7_aa-076e3472.pth',
input_size=(3, 600, 600), pool_size=(19, 19), crop_pct=0.949),
}
_DEBUG = False
_BN_MOMENTUM_PT_DEFAULT = 0.1
_BN_EPS_PT_DEFAULT = 1e-5
_BN_ARGS_PT = dict(momentum=_BN_MOMENTUM_PT_DEFAULT, eps=_BN_EPS_PT_DEFAULT)
 _BN_MOMENTUM_TF_DEFAULT = 1 - 0.99
_BN_EPS_TF_DEFAULT = 1e-3
_BN_ARGS_TF = dict(momentum=_BN_MOMENTUM_TF_DEFAULT, eps=_BN_EPS_TF_DEFAULT)
 def _initialize_weight_goog(m):
if isinstance(m, nn.Conv2d):
n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
m.weight.data.normal_(0, math.sqrt(2.0 / n))
if m.bias is not None:
m.bias.data.zero_()
elif isinstance(m, nn.BatchNorm2d):
m.weight.data.fill_(1.0)
 m.bias.data.zero_()
elif isinstance(m, nn.Linear):
n = m.weight.size(0)
 init_range = 1.0 / math.sqrt(n)
 m.weight.data.uniform_(-init_range, init_range)
 m.bias.data.zero_()
def swish(x, inplace=False):
if inplace:
return x.mul_(x.sigmoid())
 else:
return x * x.sigmoid()
def sigmoid(x, inplace=False):
return x.sigmoid_() if inplace else x.sigmoid()
class InvertedResidual(nn.Module):

 def __init__(self, in_chs, out_chs, dw_kernel_size=3,
stride=1, pad_type='', act_fn=F.relu, noskip=False,
exp_ratio=1.0, exp_kernel_size=1, pw_kernel_size=1,
se_ratio=0., se_reduce_mid=False, se_gate_fn=sigmoid,
shuffle_type=None, bn_args=_BN_ARGS_PT, drop_connect_rate=0.) :
super(InvertedResidual, self ).__init__()
mid_chs = int(in_chs * exp_ratio)
 self.has_se = se_ratio is not None and se_ratio > 0.
 self.has_residual =(in_chs == out_chs and stride == 1)and not noskip
self.act_fn = act_fn
self.drop_connect_rate = drop_connect_rate
self.conv_pw = select_conv2d(in_chs, mid_chs, exp_kernel_size, padding=pad_type)
 self.bn1 = nn.BatchNorm2d(mid_chs, **bn_args)
 self.shuffle_type = shuffle_type
if shuffle_type is not None and isinstance(exp_kernel_size, list):
self.shuffle = ChannelShuffle(len(exp_kernel_size))
self.conv_dw = select_conv2d(
 mid_chs, mid_chs, dw_kernel_size, stride=stride, padding=pad_type, depthwise=True)
 self.bn2 = nn.BatchNorm2d(mid_chs, **bn_args)
 if self.has_se:
se_base_chs = mid_chs if se_reduce_mid else in_chs
self.se = SqueezeExcite(
 mid_chs, reduce_chs=max(1, int(se_base_chs * se_ratio)) , act_fn=act_fn, gate_fn=se_gate_fn)
 self.conv_pwl = select_conv2d(mid_chs, out_chs, pw_kernel_size, padding=pad_type)
 self.bn3 = nn.BatchNorm2d(out_chs, **bn_args)
 def forward(self, x):
residual = x
x = self.conv_pw(x)
 x = self.bn1(x)
 x = self.act_fn(x, inplace=True)
 if self.shuffle_type == "mid":
x = self.shuffle(x)
 x = self.conv_dw(x)
 x = self.bn2(x)
 x = self.act_fn(x, inplace=True)
 if self.has_se:
x = self.se(x)
 x = self.conv_pwl(x)
 x = self.bn3(x)
 if self.has_residual:
if self.drop_connect_rate > 0.:
x = drop_connect(x, self.training, self.drop_connect_rate)
 x += residual
return x
class GenEfficientNet(nn.Module):

 def __init__(self, block_args, num_classes=1000, in_chans=3, stem_size=32, num_features=1280,
channel_multiplier=1.0, channel_divisor=8, channel_min=None,
pad_type='', act_fn=F.relu, drop_rate=0., drop_connect_rate=0.,
se_gate_fn=sigmoid, se_reduce_mid=False, bn_args=_BN_ARGS_PT,
global_pool='avg', head_conv='default', weight_init='goog'):
super(GenEfficientNet, self ).__init__()
self.num_classes = num_classes
self.drop_rate = drop_rate
self.act_fn = act_fn
self.num_features = num_features
stem_size = _round_channels(stem_size, channel_multiplier, channel_divisor, channel_min)
 self.conv_stem = select_conv2d(in_chans, stem_size, 3, stride=2, padding=pad_type)
 self.bn1 = nn.BatchNorm2d(stem_size, **bn_args)
 in_chs = stem_size
builder = _BlockBuilder(
 channel_multiplier, channel_divisor, channel_min,
pad_type, act_fn, se_gate_fn, se_reduce_mid,
bn_args, drop_connect_rate, verbose=_DEBUG)
 self.blocks = nn.Sequential(*builder(in_chs, block_args))
in_chs = builder.in_chs
if not head_conv or head_conv == 'none':
self.efficient_head = False
self.conv_head = None
assert in_chs == self.num_features
else:
self.efficient_head = head_conv == 'efficient'
self.conv_head = select_conv2d(in_chs, self.num_features, 1, padding=pad_type)
 self.bn2 = None if self.efficient_head else nn.BatchNorm2d(self.num_features, **bn_args)
 self.global_pool = SelectAdaptivePool2d(pool_type=global_pool)
 self.classifier = nn.Linear(self.num_features * self.global_pool.feat_mult() , self.num_classes)
 for m in self.modules() :
if weight_init == 'goog':
_initialize_weight_goog(m)
 else:
_initialize_weight_default(m)
 def get_classifier(self):
return self.classifier
def reset_classifier(self, num_classes, global_pool='avg'):
self.global_pool = SelectAdaptivePool2d(pool_type=global_pool)
 self.num_classes = num_classes
del self.classifier
if num_classes:
self.classifier = nn.Linear(
 self.num_features * self.global_pool.feat_mult() , num_classes)
 else:
self.classifier = None
def forward_features(self, x, pool=True):
x = self.conv_stem(x)
 x = self.bn1(x)
 x = self.act_fn(x, inplace=True)
 x = self.blocks(x)
 if self.efficient_head:
x = self.global_pool(x)
 x = self.conv_head(x)
 x = self.act_fn(x, inplace=True)
 if pool:
x = x.view(x.size(0), -1)
 else:
if self.conv_head is not None:
x = self.conv_head(x)
 x = self.bn2(x)
 x = self.act_fn(x, inplace=True)
 if pool:
x = self.global_pool(x)
 x = x.view(x.size(0), -1)
 return x
def forward(self, x):
x = self.forward_features(x)
 if self.drop_rate > 0.:
x = F.dropout(x, p=self.drop_rate, training=self.training)
 return self.classifier(x)
 class DepthwiseSeparableConv(nn.Module):

 def __init__(self, in_chs, out_chs, dw_kernel_size=3,
stride=1, pad_type='', act_fn=F.relu, noskip=False,
pw_kernel_size=1, pw_act=False,
se_ratio=0., se_gate_fn=sigmoid,
bn_args=_BN_ARGS_PT, drop_connect_rate=0.) :
super(DepthwiseSeparableConv, self ).__init__()
assert stride in [1, 2]
self.has_se = se_ratio is not None and se_ratio > 0.
 self.has_residual =(stride == 1 and in_chs == out_chs)and not noskip
self.has_pw_act = pw_act
self.act_fn = act_fn
self.drop_connect_rate = drop_connect_rate
self.conv_dw = select_conv2d(
 in_chs, in_chs, dw_kernel_size, stride=stride, padding=pad_type, depthwise=True)
 self.bn1 = nn.BatchNorm2d(in_chs, **bn_args)
 if self.has_se:
self.se = SqueezeExcite(
 in_chs, reduce_chs=max(1, int(in_chs * se_ratio)) , act_fn=act_fn, gate_fn=se_gate_fn)
 self.conv_pw = select_conv2d(in_chs, out_chs, pw_kernel_size, padding=pad_type)
 self.bn2 = nn.BatchNorm2d(out_chs, **bn_args)
 def forward(self, x):
residual = x
x = self.conv_dw(x)
 x = self.bn1(x)
 x = self.act_fn(x, inplace=True)
 if self.has_se:
x = self.se(x)
 x = self.conv_pw(x)
 x = self.bn2(x)
 if self.has_pw_act:
x = self.act_fn(x, inplace=True)
 if self.has_residual:
if self.drop_connect_rate > 0.:
x = drop_connect(x, self.training, self.drop_connect_rate)
 x += residual
return x
class SqueezeExcite(nn.Module):
def __init__(self, in_chs, reduce_chs=None, act_fn=F.relu, gate_fn=sigmoid):
super(SqueezeExcite, self ).__init__()
self.act_fn = act_fn
self.gate_fn = gate_fn
reduced_chs = reduce_chs or in_chs
self.conv_reduce = nn.Conv2d(in_chs, reduced_chs, 1, bias=True)
 self.conv_expand = nn.Conv2d(reduced_chs, in_chs, 1, bias=True)
 def forward(self, x):
x_se = x.view(x.size(0), x.size(1), -1 ).mean(-1 ).view(x.size(0), x.size(1), 1, 1)
 x_se = self.conv_reduce(x_se)
 x_se = self.act_fn(x_se, inplace=True)
 x_se = self.conv_expand(x_se)
 x = x * self.gate_fn(x_se)
 return x
class _BlockBuilder:

 def __init__(self, channel_multiplier=1.0, channel_divisor=8, channel_min=None,
pad_type='', act_fn=None, se_gate_fn=sigmoid, se_reduce_mid=False,
bn_args=_BN_ARGS_PT, drop_connect_rate=0., verbose=False):
self.channel_multiplier = channel_multiplier
self.channel_divisor = channel_divisor
self.channel_min = channel_min
self.pad_type = pad_type
self.act_fn = act_fn
self.se_gate_fn = se_gate_fn
self.se_reduce_mid = se_reduce_mid
self.bn_args = bn_args
self.drop_connect_rate = drop_connect_rate
self.verbose = verbose
self.in_chs = None
self.block_idx = 0
self.block_count = 0
def _round_channels(self, chs):
return _round_channels(chs, self.channel_multiplier, self.channel_divisor, self.channel_min)
 def _make_block(self, ba):
bt = ba.pop('block_type')
 ba['in_chs'] = self.in_chs
ba['out_chs'] = self._round_channels(ba['out_chs'])
 ba['bn_args'] = self.bn_args
ba['pad_type'] = self.pad_type
ba['act_fn'] = ba['act_fn'] if ba['act_fn'] is not None else self.act_fn
assert ba['act_fn'] is not None
if bt == 'ir':
ba['drop_connect_rate'] = self.drop_connect_rate * self.block_idx / self.block_count
ba['se_gate_fn'] = self.se_gate_fn
ba['se_reduce_mid'] = self.se_reduce_mid
if self.verbose:
logging.info(' InvertedResidual {}, Args: {}'.format(self.block_idx, str(ba)))
 block = InvertedResidual(**ba)
 elif bt == 'ds' or bt == 'dsa':
ba['drop_connect_rate'] = self.drop_connect_rate * self.block_idx / self.block_count
if self.verbose:
logging.info(' DepthwiseSeparable {}, Args: {}'.format(self.block_idx, str(ba)))
 block = DepthwiseSeparableConv(**ba)
 elif bt == 'cn':
if self.verbose:
logging.info(' ConvBnAct {}, Args: {}'.format(self.block_idx, str(ba)))
 block = ConvBnAct(**ba)
 else:
assert False, 'Uknkown block type(%s)while building model.' % bt
self.in_chs = ba['out_chs']
return block
def _make_stack(self, stack_args):
blocks = []
for i, ba in enumerate(stack_args):
if self.verbose:
logging.info(' Block: {}'.format(i))
if i >= 1:
ba['stride'] = 1
block = self._make_block(ba)
 blocks.append(block)
 self.block_idx += 1
return nn.Sequential(*blocks)
 def __call__(self, in_chs, block_args):

 if self.verbose:
logging.info('Building model trunk with %d stages...' % len(block_args))
self.in_chs = in_chs
self.block_count = sum([len(x)for x in block_args])
 self.block_idx = 0
blocks = []
for stack_idx, stack in enumerate(block_args):
if self.verbose:
logging.info('Stack: {}'.format(stack_idx))
assert isinstance(stack, list)
 stack = self._make_stack(stack)
 blocks.append(stack)
 return blocks
def _gen_efficientnet(channel_multiplier=1.0, depth_multiplier=1.0, num_classes=1000, **kwargs):

 arch_def = [
['ds_r1_k3_s1_e1_c16_se0.25'],
['ir_r2_k3_s2_e6_c24_se0.25'],
['ir_r2_k5_s2_e6_c40_se0.25'],
['ir_r3_k3_s2_e6_c80_se0.25'],
['ir_r3_k5_s1_e6_c112_se0.25'],
['ir_r4_k5_s2_e6_c192_se0.25'],
['ir_r1_k3_s1_e6_c320_se0.25'],
]
num_features = _round_channels(1280, channel_multiplier, 8, None)
 model = GenEfficientNet(
 _decode_arch_def(arch_def, depth_multiplier),
num_classes=num_classes,
stem_size=32,
channel_multiplier=channel_multiplier,
num_features=num_features,
bn_args=_resolve_bn_args(kwargs),
act_fn=swish,
**kwargs 
)
 return model
def tf_efficientnet_b0(pretrained=False, num_classes=1000, in_chans=3, **kwargs):

 default_cfg = default_cfgs['tf_efficientnet_b0']
kwargs['bn_eps'] = _BN_EPS_TF_DEFAULT
kwargs['pad_type'] = 'same'
model = _gen_efficientnet(
 channel_multiplier=1.0, depth_multiplier=1.0,
num_classes=num_classes, in_chans=in_chans, **kwargs)
 model.default_cfg = default_cfg
if pretrained:
load_pretrained(model, default_cfg, num_classes, in_chans)
 return model
def tf_efficientnet_b1(pretrained=False, num_classes=1000, in_chans=3, **kwargs):

 default_cfg = default_cfgs['tf_efficientnet_b1']
kwargs['bn_eps'] = _BN_EPS_TF_DEFAULT
kwargs['pad_type'] = 'same'
model = _gen_efficientnet(
 channel_multiplier=1.0, depth_multiplier=1.1,
num_classes=num_classes, in_chans=in_chans, **kwargs)
 model.default_cfg = default_cfg
if pretrained:
load_pretrained(model, default_cfg, num_classes, in_chans)
 return model
def tf_efficientnet_b2(pretrained=False, num_classes=1000, in_chans=3, **kwargs):

 default_cfg = default_cfgs['tf_efficientnet_b2']
kwargs['bn_eps'] = _BN_EPS_TF_DEFAULT
kwargs['pad_type'] = 'same'
model = _gen_efficientnet(
 channel_multiplier=1.1, depth_multiplier=1.2,
num_classes=num_classes, in_chans=in_chans, **kwargs)
 model.default_cfg = default_cfg
if pretrained:
load_pretrained(model, default_cfg, num_classes, in_chans)
 return model
def tf_efficientnet_b3(pretrained=False, num_classes=1000, in_chans=3, **kwargs):

 default_cfg = default_cfgs['tf_efficientnet_b3']
kwargs['bn_eps'] = _BN_EPS_TF_DEFAULT
kwargs['pad_type'] = 'same'
model = _gen_efficientnet(
 channel_multiplier=1.2, depth_multiplier=1.4,
num_classes=num_classes, in_chans=in_chans, **kwargs)
 model.default_cfg = default_cfg
if pretrained:
load_pretrained(model, default_cfg, num_classes, in_chans)
 return model
def tf_efficientnet_b4(pretrained=False, num_classes=1000, in_chans=3, **kwargs):

 default_cfg = default_cfgs['tf_efficientnet_b4']
kwargs['bn_eps'] = _BN_EPS_TF_DEFAULT
kwargs['pad_type'] = 'same'
model = _gen_efficientnet(
 channel_multiplier=1.4, depth_multiplier=1.8,
num_classes=num_classes, in_chans=in_chans, **kwargs)
 model.default_cfg = default_cfg
if pretrained:
load_pretrained(model, default_cfg, num_classes, in_chans)
 return model
def tf_efficientnet_b5(pretrained=False, num_classes=1000, in_chans=3, **kwargs):

 default_cfg = default_cfgs['tf_efficientnet_b5']
kwargs['bn_eps'] = _BN_EPS_TF_DEFAULT
kwargs['pad_type'] = 'same'
model = _gen_efficientnet(
 channel_multiplier=1.6, depth_multiplier=2.2,
num_classes=num_classes, in_chans=in_chans, **kwargs)
 model.default_cfg = default_cfg
if pretrained:
load_pretrained(model, default_cfg, num_classes, in_chans)
 return model
def tf_efficientnet_b6(pretrained=False, num_classes=1000, in_chans=3, **kwargs):

 default_cfg = default_cfgs['tf_efficientnet_b6']
kwargs['bn_eps'] = _BN_EPS_TF_DEFAULT
kwargs['pad_type'] = 'same'
model = _gen_efficientnet(
 channel_multiplier=1.8, depth_multiplier=2.6,
num_classes=num_classes, in_chans=in_chans, **kwargs)
 model.default_cfg = default_cfg
if pretrained:
load_pretrained(model, default_cfg, num_classes, in_chans)
 return model
def tf_efficientnet_b7(pretrained=False, num_classes=1000, in_chans=3, **kwargs):

 default_cfg = default_cfgs['tf_efficientnet_b7']
kwargs['bn_eps'] = _BN_EPS_TF_DEFAULT
kwargs['pad_type'] = 'same'
model = _gen_efficientnet(
 channel_multiplier=2.0, depth_multiplier=3.1,
num_classes=num_classes, in_chans=in_chans, **kwargs)
 model.default_cfg = default_cfg
if pretrained:
load_pretrained(model, default_cfg, num_classes, in_chans)
 return model<import_modules>
shuffle_and_test(20,30,X_At4_val )
from fastai import *
from fastai.vision import *
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
import os
import scipy as sp
from functools import partial
from sklearn import metrics
from collections import Counter
from fastai.callbacks import *
import PIL
import cv2<set_options>
model1=create_simple_mlp()
model1.compile(loss='sparse_categorical_crossentropy', optimizer=opt, metrics=['acc'])
 history_model1 = model1.fit(X_At2_train,y_At2_train ,batch_size=16, validation_data=(X_At4_val, y_At4_val), epochs=400, callbacks=[es],verbose=0)
 plot_accuracy_and_loss(history_model1)
def seed_everything(seed=77):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything()<define_variables>
pred_at2=model1.predict(X_At4_val )
LOCAL = False<load_from_csv>
model2 = Sequential()
for layer in model1.layers[:-1]:
model2.add(layer)
 model2.add(Dense(30, activation='selu'))
model2.add(Dense(10, activation='selu'))
model2.add(Dense(4, activation='softmax'))
def get_df() :
if LOCAL:
base_image_dir = os.path.join('.', 'data/')
 else:
base_image_dir = os.path.join('.. ', 'input/aptos2019-blindness-detection/')
 train_dir = os.path.join(base_image_dir,'train_images/')
 df = pd.read_csv(os.path.join(base_image_dir, 'train.csv'))
df['path'] = df['id_code'].map(lambda x: os.path.join(train_dir,'{}.png'.format(x)))
 df = df.drop(columns=['id_code'])
 df = df.sample(frac=1 ).reset_index(drop=True)
 test_df = pd.read_csv(os.path.join(base_image_dir, 'sample_submission.csv'))
return df, test_df
df, test_df = get_df()<compute_test_metric>
model2.compile(loss='sparse_categorical_crossentropy', optimizer=opt, metrics=['acc'])
 history_model2 = model2.fit(X_At4_train,y_At4_train ,batch_size=64, validation_data=(X_At4_val, y_At4_val), epochs=100, callbacks=[es],verbose=0)
 plot_accuracy_and_loss(history_model2)
def qk(y_pred, y):
return torch.tensor(cohen_kappa_score(torch.round(y_pred), y,
weights='quadratic'), device='cuda:0' )<install_modules>
def calculate_score(df):
output=pd.DataFrame()
for i in range(0,10):
sum_spikes=df.iloc[:,i]
latency=df.iloc[:,i+10] + 1.5
output[i]= sum_spikes/latency
for i in range(0,10):
output[i+10]=df.iloc[:,i+20]
return output
!mkdir models
!cp '.. /input/effinet-b5-aptos/Effi0_12e_pretrained.pth' "models/"
!cp '.. /input/effinet-b5-aptos/Effi1_12e_pretrained.pth' "models/"
!cp '.. /input/effinet-b5-aptos/Effi2_10e_pretrained.pth' "models/"
!cp '.. /input/effinet-b5-aptos/Effi3_10e_pretrained.pth' "models/"
!cp '.. /input/effinet-b5-aptos/best_effi6_224.pth' "models/"
!cp '.. /input/starter-kernel-for-0-79/models/abcdef.pth' "models/"<categorify>
df_copy=at4_train.copy()
train_values_to10 = df_copy.iloc[:,1:11].values
train_values_to20 = df_copy.iloc[:,11:21].values
train_values_to30= df_copy.iloc[:,21:31].values
min_max_scaler_spike = preprocessing.MinMaxScaler(feature_range=(0,1))
min_max_scaler_latency = preprocessing.MinMaxScaler(feature_range=(0,1))
min_max_scaler_rank = preprocessing.MinMaxScaler(feature_range=(0,1))
transformation_to10 = min_max_scaler_spike.fit(train_values_to10.T)
 train_values_scaled_10 = min_max_scaler_spike.transform(train_values_to10.T ).T
transformation_to20 = min_max_scaler_latency.fit(train_values_to20.T)
 train_values_scaled_20 = min_max_scaler_latency.transform(train_values_to20.T ).T
transformation_to30 = min_max_scaler_rank.fit(train_values_to30.T)
 train_values_scaled_30 = min_max_scaler_rank.transform(train_values_to30.T ).T
train_scaled=pd.DataFrame()
train_scaled['label']=at4_train['label']
for i in range(1,11):
train_scaled['onsc'+str(i)]=pd.DataFrame(train_values_scaled_10 ).iloc[:,i-1]
for i in range(1,11):
train_scaled['onstf'+str(i)]=pd.DataFrame(train_values_scaled_20 ).iloc[:,i-1]
for i in range(1,11):
train_scaled['onsto'+str(i)]=pd.DataFrame(train_values_scaled_30 ).iloc[:,i-1]
data2 =(ImageList.from_df(df=df,path='./',cols='path')
.split_by_rand_pct(0.2)
.label_from_df(cols='diagnosis',label_cls=FloatList)
.transform(tfms,size=sz,resize_method=ResizeMethod.SQUISH,padding_mode='zeros')
.databunch(bs=bs)
.normalize(imagenet_stats)
)
 if not LOCAL:
data2.add_test(ImageList.from_df(test_df, '.. /input/aptos2019-blindness-detection/',
folder='test_images',
suffix='.png'))
bs = 32
sz = 224
tfms = get_transforms(do_flip=True,flip_vert=True)
 learn2 = Learner(data2,
tf_efficientnet_b2(pretrained=False, num_classes=1),
metrics = [qk],
model_dir="models" ).to_fp16()
learn2.load('Effi2_10e_pretrained');
preds2,y = learn2.get_preds(DatasetType.Test)
 del learn2, data2<set_options>
X_train_30, X_val_30, y_train, y_val=train_test_split(train_scaled.loc[: , train_scaled.columns!='label'],train_scaled['label'],random_state=0,test_size=0.2 )
gc.collect() ;<set_options>
X_train=calculate_score(X_train_30)
 X_val=calculate_score(X_val_30 )
gc.collect() ;<categorify>
y_cross_validation=train_scaled.iloc[:,0]
X_cross_validation=calculate_score(train_scaled.iloc[:,1:] )
GlobalParams = collections.namedtuple('GlobalParams', [
'batch_norm_momentum', 'batch_norm_epsilon', 'dropout_rate',
'num_classes', 'width_coefficient', 'depth_coefficient',
'depth_divisor', 'min_depth', 'drop_connect_rate', 'image_size'])
 BlockArgs = collections.namedtuple('BlockArgs', [
'kernel_size', 'num_repeat', 'input_filters', 'output_filters',
'expand_ratio', 'id_skip', 'stride', 'se_ratio'])
 GlobalParams.__new__.__defaults__ =(None,)* len(GlobalParams._fields)
 BlockArgs.__new__.__defaults__ =(None,)* len(BlockArgs._fields)
 def relu_fn(x):

 return x * torch.sigmoid(x)
 def round_filters(filters, global_params):

 multiplier = global_params.width_coefficient
if not multiplier:
return filters
divisor = global_params.depth_divisor
min_depth = global_params.min_depth
filters *= multiplier
min_depth = min_depth or divisor
new_filters = max(min_depth, int(filters + divisor / 2)// divisor * divisor)
 if new_filters < 0.9 * filters:
new_filters += divisor
return int(new_filters)
 def round_repeats(repeats, global_params):

 multiplier = global_params.depth_coefficient
if not multiplier:
return repeats
return int(math.ceil(multiplier * repeats))
def drop_connect(inputs, p, training):

 if not training: return inputs
batch_size = inputs.shape[0]
keep_prob = 1 - p
random_tensor = keep_prob
random_tensor += torch.rand([batch_size, 1, 1, 1], dtype=inputs.dtype, device=inputs.device)
 binary_tensor = torch.floor(random_tensor)
 output = inputs / keep_prob * binary_tensor
return output
def get_same_padding_conv2d(image_size=None):

 if image_size is None:
return Conv2dDynamicSamePadding
else:
return partial(Conv2dStaticSamePadding, image_size=image_size)
 class Conv2dDynamicSamePadding(nn.Conv2d):

 def __init__(self, in_channels, out_channels, kernel_size, stride=1, dilation=1, groups=1, bias=True):
super().__init__(in_channels, out_channels, kernel_size, stride, 0, dilation, groups, bias)
 self.stride = self.stride if len(self.stride)== 2 else [self.stride[0]]*2
def forward(self, x):
ih, iw = x.size() [-2:]
kh, kw = self.weight.size() [-2:]
sh, sw = self.stride
oh, ow = math.ceil(ih / sh), math.ceil(iw / sw)
 pad_h = max(( oh - 1)* self.stride[0] +(kh - 1)* self.dilation[0] + 1 - ih, 0)
 pad_w = max(( ow - 1)* self.stride[1] +(kw - 1)* self.dilation[1] + 1 - iw, 0)
 if pad_h > 0 or pad_w > 0:
x = F.pad(x, [pad_w//2, pad_w - pad_w//2, pad_h//2, pad_h - pad_h//2])
 return F.conv2d(x, self.weight, self.bias, self.stride, self.padding, self.dilation, self.groups)
 class Conv2dStaticSamePadding(nn.Conv2d):

 def __init__(self, in_channels, out_channels, kernel_size, image_size=None, **kwargs):
super().__init__(in_channels, out_channels, kernel_size, **kwargs)
 self.stride = self.stride if len(self.stride)== 2 else [self.stride[0]] * 2
assert image_size is not None
ih, iw = image_size if type(image_size)== list else [image_size, image_size]
kh, kw = self.weight.size() [-2:]
sh, sw = self.stride
oh, ow = math.ceil(ih / sh), math.ceil(iw / sw)
 pad_h = max(( oh - 1)* self.stride[0] +(kh - 1)* self.dilation[0] + 1 - ih, 0)
 pad_w = max(( ow - 1)* self.stride[1] +(kw - 1)* self.dilation[1] + 1 - iw, 0)
 if pad_h > 0 or pad_w > 0:
self.static_padding = nn.ZeroPad2d(( pad_w // 2, pad_w - pad_w // 2, pad_h // 2, pad_h - pad_h // 2))
else:
self.static_padding = Identity()
def forward(self, x):
x = self.static_padding(x)
 x = F.conv2d(x, self.weight, self.bias, self.stride, self.padding, self.dilation, self.groups)
 return x
class Identity(nn.Module):
def __init__(self,):
super(Identity, self ).__init__()
def forward(self, input):
return input
def efficientnet_params(model_name):

 params_dict = {
'efficientnet-b0':(1.0, 1.0, 224, 0.2),
'efficientnet-b1':(1.0, 1.1, 240, 0.2),
'efficientnet-b2':(1.1, 1.2, 260, 0.3),
'efficientnet-b3':(1.2, 1.4, 300, 0.3),
'efficientnet-b4':(1.4, 1.8, 380, 0.4),
'efficientnet-b5':(1.6, 2.2, 456, 0.4),
'efficientnet-b6':(1.8, 2.6, 528, 0.5),
'efficientnet-b7':(2.0, 3.1, 600, 0.5),
}
return params_dict[model_name]
class BlockDecoder(object):

 @staticmethod
def _decode_block_string(block_string):

 assert isinstance(block_string, str)
 ops = block_string.split('_')
 options = {}
for op in ops:
splits = re.split(r'(\d.*)', op)
 if len(splits)>= 2:
key, value = splits[:2]
options[key] = value
assert(( 's' in options and len(options['s'])== 1)or 
(len(options['s'])== 2 and options['s'][0] == options['s'][1]))
return BlockArgs(
 kernel_size=int(options['k']),
num_repeat=int(options['r']),
input_filters=int(options['i']),
output_filters=int(options['o']),
expand_ratio=int(options['e']),
id_skip=('noskip' not in block_string),
se_ratio=float(options['se'])if 'se' in options else None,
stride=[int(options['s'][0])])
 @staticmethod
def _encode_block_string(block):

 args = [
'r%d' % block.num_repeat,
'k%d' % block.kernel_size,
's%d%d' %(block.strides[0], block.strides[1]),
'e%s' % block.expand_ratio,
'i%d' % block.input_filters,
'o%d' % block.output_filters
]
if 0 < block.se_ratio <= 1:
args.append('se%s' % block.se_ratio)
 if block.id_skip is False:
args.append('noskip')
 return '_'.join(args)
 @staticmethod
def decode(string_list):

 assert isinstance(string_list, list)
 blocks_args = []
for block_string in string_list:
blocks_args.append(BlockDecoder._decode_block_string(block_string))
return blocks_args
@staticmethod
def encode(blocks_args):

 block_strings = []
for block in blocks_args:
block_strings.append(BlockDecoder._encode_block_string(block))
return block_strings
def efficientnet(width_coefficient=None, depth_coefficient=None, dropout_rate=0.2,
drop_connect_rate=0.2, image_size=None, num_classes=1000):

 blocks_args = [
'r1_k3_s11_e1_i32_o16_se0.25', 'r2_k3_s22_e6_i16_o24_se0.25',
'r2_k5_s22_e6_i24_o40_se0.25', 'r3_k3_s22_e6_i40_o80_se0.25',
'r3_k5_s11_e6_i80_o112_se0.25', 'r4_k5_s22_e6_i112_o192_se0.25',
'r1_k3_s11_e6_i192_o320_se0.25',
]
blocks_args = BlockDecoder.decode(blocks_args)
 global_params = GlobalParams(
 batch_norm_momentum=0.99,
batch_norm_epsilon=1e-3,
dropout_rate=dropout_rate,
drop_connect_rate=drop_connect_rate,
num_classes=num_classes,
width_coefficient=width_coefficient,
depth_coefficient=depth_coefficient,
depth_divisor=8,
min_depth=None,
image_size=image_size, 
)
 return blocks_args, global_params
def get_model_params(model_name, override_params):

 if model_name.startswith('efficientnet'):
w, d, s, p = efficientnet_params(model_name)
 blocks_args, global_params = efficientnet(
 width_coefficient=w, depth_coefficient=d, dropout_rate=p, image_size=s)
 else:
raise NotImplementedError('model name is not pre-defined: %s' % model_name)
 if override_params:
global_params = global_params._replace(**override_params)
 return blocks_args, global_params
url_map = {
'efficientnet-b0': 'http://storage.googleapis.com/public-models/efficientnet-b0-08094119.pth',
'efficientnet-b1': 'http://storage.googleapis.com/public-models/efficientnet-b1-dbc7070a.pth',
'efficientnet-b2': 'http://storage.googleapis.com/public-models/efficientnet-b2-27687264.pth',
'efficientnet-b3': 'http://storage.googleapis.com/public-models/efficientnet-b3-c8376fa2.pth',
'efficientnet-b4': 'http://storage.googleapis.com/public-models/efficientnet-b4-e116e8b3.pth',
'efficientnet-b5': 'http://storage.googleapis.com/public-models/efficientnet-b5-586e6cc6.pth',
}
def load_pretrained_weights(model, model_name, load_fc=True):

 state_dict = model_zoo.load_url(url_map[model_name])
 if load_fc:
model.load_state_dict(state_dict)
 else:
state_dict.pop('_fc.weight')
 state_dict.pop('_fc.bias')
 res = model.load_state_dict(state_dict, strict=False)
 assert str(res.missing_keys)== str(['_fc.weight', '_fc.bias']), 'issue loading pretrained weights'
print('Loaded pretrained weights for {}'.format(model_name))
class MBConvBlock(nn.Module):

 def __init__(self, block_args, global_params):
super().__init__()
self._block_args = block_args
self._bn_mom = 1 - global_params.batch_norm_momentum
self._bn_eps = global_params.batch_norm_epsilon
self.has_se =(self._block_args.se_ratio is not None)and(0 < self._block_args.se_ratio <= 1)
 self.id_skip = block_args.id_skip
Conv2d = get_same_padding_conv2d(image_size=global_params.image_size)
 inp = self._block_args.input_filters
oup = self._block_args.input_filters * self._block_args.expand_ratio
if self._block_args.expand_ratio != 1:
self._expand_conv = Conv2d(in_channels=inp, out_channels=oup, kernel_size=1, bias=False)
 self._bn0 = nn.BatchNorm2d(num_features=oup, momentum=self._bn_mom, eps=self._bn_eps)
 k = self._block_args.kernel_size
s = self._block_args.stride
self._depthwise_conv = Conv2d(
 in_channels=oup, out_channels=oup, groups=oup,
kernel_size=k, stride=s, bias=False)
 self._bn1 = nn.BatchNorm2d(num_features=oup, momentum=self._bn_mom, eps=self._bn_eps)
 if self.has_se:
num_squeezed_channels = max(1, int(self._block_args.input_filters * self._block_args.se_ratio))
self._se_reduce = Conv2d(in_channels=oup, out_channels=num_squeezed_channels, kernel_size=1)
 self._se_expand = Conv2d(in_channels=num_squeezed_channels, out_channels=oup, kernel_size=1)
 final_oup = self._block_args.output_filters
self._project_conv = Conv2d(in_channels=oup, out_channels=final_oup, kernel_size=1, bias=False)
 self._bn2 = nn.BatchNorm2d(num_features=final_oup, momentum=self._bn_mom, eps=self._bn_eps)
 def forward(self, inputs, drop_connect_rate=None):

 x = inputs
if self._block_args.expand_ratio != 1:
x = relu_fn(self._bn0(self._expand_conv(inputs)))
 x = relu_fn(self._bn1(self._depthwise_conv(x)))
 if self.has_se:
x_squeezed = F.adaptive_avg_pool2d(x, 1)
 x_squeezed = self._se_expand(relu_fn(self._se_reduce(x_squeezed)))
 x = torch.sigmoid(x_squeezed)* x
x = self._bn2(self._project_conv(x))
input_filters, output_filters = self._block_args.input_filters, self._block_args.output_filters
if self.id_skip and self._block_args.stride == 1 and input_filters == output_filters:
if drop_connect_rate:
x = drop_connect(x, p=drop_connect_rate, training=self.training)
 x = x + inputs
return x
class EfficientNet(nn.Module):

 def __init__(self, blocks_args=None, global_params=None):
super().__init__()
assert isinstance(blocks_args, list), 'blocks_args should be a list'
assert len(blocks_args)> 0, 'block args must be greater than 0'
self._global_params = global_params
self._blocks_args = blocks_args
Conv2d = get_same_padding_conv2d(image_size=global_params.image_size)
 bn_mom = 1 - self._global_params.batch_norm_momentum
bn_eps = self._global_params.batch_norm_epsilon
in_channels = 3
out_channels = round_filters(32, self._global_params)
 self._conv_stem = Conv2d(in_channels, out_channels, kernel_size=3, stride=2, bias=False)
 self._bn0 = nn.BatchNorm2d(num_features=out_channels, momentum=bn_mom, eps=bn_eps)
 self._blocks = nn.ModuleList([])
 for block_args in self._blocks_args:
block_args = block_args._replace(
 input_filters=round_filters(block_args.input_filters, self._global_params),
output_filters=round_filters(block_args.output_filters, self._global_params),
num_repeat=round_repeats(block_args.num_repeat, self._global_params)
)
 self._blocks.append(MBConvBlock(block_args, self._global_params))
if block_args.num_repeat > 1:
block_args = block_args._replace(input_filters=block_args.output_filters, stride=1)
 for _ in range(block_args.num_repeat - 1):
self._blocks.append(MBConvBlock(block_args, self._global_params))
in_channels = block_args.output_filters
out_channels = round_filters(1280, self._global_params)
 self._conv_head = Conv2d(in_channels, out_channels, kernel_size=1, bias=False)
 self._bn1 = nn.BatchNorm2d(num_features=out_channels, momentum=bn_mom, eps=bn_eps)
 self._dropout = self._global_params.dropout_rate
self._fc = nn.Linear(out_channels, self._global_params.num_classes)
 def extract_features(self, inputs):

 x = relu_fn(self._bn0(self._conv_stem(inputs)))
 for idx, block in enumerate(self._blocks):
drop_connect_rate = self._global_params.drop_connect_rate
if drop_connect_rate:
drop_connect_rate *= float(idx)/ len(self._blocks)
 x = block(x, drop_connect_rate=drop_connect_rate)
 x = relu_fn(self._bn1(self._conv_head(x)))
 return x
def forward(self, inputs):

 x = self.extract_features(inputs)
 x = F.adaptive_avg_pool2d(x, 1 ).squeeze(-1 ).squeeze(-1)
 if self._dropout:
x = F.dropout(x, p=self._dropout, training=self.training)
 x = self._fc(x)
 return x
@classmethod
def from_name(cls, model_name, override_params=None):
cls._check_model_name_is_valid(model_name)
 blocks_args, global_params = get_model_params(model_name, override_params)
 return EfficientNet(blocks_args, global_params)
 @classmethod
def from_pretrained(cls, model_name, num_classes=1000):
model = EfficientNet.from_name(model_name, override_params={'num_classes': num_classes})
 return model
@classmethod
def get_image_size(cls, model_name):
cls._check_model_name_is_valid(model_name)
 _, _, res, _ = efficientnet_params(model_name)
 return res
@classmethod
def _check_model_name_is_valid(cls, model_name, also_need_pretrained_weights=False):

 num_models = 4 if also_need_pretrained_weights else 8
valid_models = ['efficientnet_b'+str(i)for i in range(num_models)]
if model_name.replace('-','_')not in valid_models:
raise ValueError('model_name should be one of: ' + ', '.join(valid_models)) 
<normalization>
def create_model() :
model = Sequential()
model.add(Dense(30,input_dim=20, activation='swish'))
model.add(Dropout(0.1))
model.add(Dense(50,activation='swish'))
model.add(Dropout(0.1))
model.add(Dense(30,activation='swish'))
model.add(Dropout(0.1))
model.add(Dense(10,activation='swish'))
model.add(Dropout(0.1))
model.add(Dense(4, activation='softmax'))
opt = keras.optimizers.Adam(learning_rate=0.004)
 model.compile(loss='sparse_categorical_crossentropy', optimizer=opt, metrics=['acc'])
 return model
bs = 64
sz = 224
tfms = get_transforms(do_flip=True,flip_vert=True)
 data4 =(ImageList.from_df(df=df,path='./',cols='path')
.split_by_rand_pct(0.2)
.label_from_df(cols='diagnosis',label_cls=FloatList)
.transform(tfms,size=sz,resize_method=ResizeMethod.SQUISH,padding_mode='zeros')
.databunch(bs=bs,num_workers=4)
.normalize(imagenet_stats)
)
 learn4 = Learner(data4,
EfficientNet.from_pretrained('efficientnet-b5', num_classes=1),
metrics = [qk],
model_dir="models" ).to_fp16()
learn4.data.add_test(ImageList.from_df(test_df,
'.. /input/aptos2019-blindness-detection',
folder='test_images',
suffix='.png'))
learn4.load('abcdef');
preds4,y = learn4.get_preds(DatasetType.Test)
 del learn4, data4<set_options>
nn=create_model()
es = EarlyStopping(monitor='val_loss', mode='auto', verbose=1,patience=25)
gc.collect() ;<compute_train_metric>
NN_clf=KerasClassifier(build_fn=create_model, epochs=200, batch_size= 16,verbose=0)
 NN_clf._estimator_type = "classifier"
class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
ll = metrics.cohen_kappa_score(y, X_p, weights='quadratic')
 return -ll
def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X=X, y=y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 print(-loss_partial(self.coef_['x']))
def predict(self, X, coef):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
return X_p
def coefficients(self):
return self.coef_['x']<compute_test_metric>
scores_cross_validation = cross_val_score(NN_clf,X_cross_validation, y_cross_validation, cv=5 )
pred =(preds1+preds2+preds3+preds4+preds5+preds6)/6<save_to_csv>
history_final = nn.fit(X_train,y_train ,batch_size=64, validation_data=(X_val, y_val), epochs=200, callbacks=[es],verbose=0)
 plot_accuracy_and_loss(history_final)
def run_subm(preds, coefficients=[0.5, 1.5, 2.5, 3.5]):
opt = OptimizedRounder()
tst_pred = opt.predict(preds, coefficients)
 test_df.diagnosis = tst_pred.astype(int)
 test_df.to_csv('submission.csv',index=False)
 print('done' )<set_options>
estimators = [('NN', NN_clf), 
('svm', BaggingClassifier(base_estimator=SVC() , n_estimators=20, random_state=0)) , 
('rforest', ExtraTreesClassifier(n_estimators=50, random_state=0,min_samples_split=4)) , 
('grad_boost',GradientBoostingClassifier(n_estimators=200)) ]
%reload_ext autoreload
%autoreload 2
%matplotlib inline
%matplotlib inline 
<load_pretrained>
estmiators_to_cross_validate=[NN_clf,BaggingClassifier(base_estimator=SVC() , n_estimators=20, random_state=0),ExtraTreesClassifier(n_estimators=50, random_state=0,min_samples_split=4),GradientBoostingClassifier(n_estimators=200)]
md_ef = EfficientNet.from_pretrained('efficientnet-b5', num_classes=1 )<load_from_csv>
reg = StackingClassifier(estimators=estimators,final_estimator=LogisticRegression())
 reg._estimator_type = "classifier"
def get_df() :
base_image_dir = os.path.join('.. ', 'input/aptos2019-blindness-detection/')
 train_dir = os.path.join(base_image_dir,'train_images/')
 df = pd.read_csv(os.path.join(base_image_dir, 'train.csv'))
df['path'] = df['id_code'].map(lambda x: os.path.join(train_dir,'{}.png'.format(x)))
 df = df.drop(columns=['id_code'])
 df = df.sample(frac=1 ).reset_index(drop=True)
 test_df = pd.read_csv('.. /input/aptos2019-blindness-detection/sample_submission.csv')
 return df, test_df
df, test_df = get_df()<feature_engineering>
for c_model in estmiators_to_cross_validate:
scores_cross_validation_reg = cross_val_score(c_model,X_cross_validation, y_cross_validation, cv=5)
 print(scores_cross_validation_reg )
bs = 32
sz = 224
tfms = get_transforms(do_flip=False,flip_vert=False )<compute_test_metric>
reg.fit(X_train,y_train ).score(X_val,y_val)
def qk(y_pred, y):
return torch.tensor(cohen_kappa_score(torch.round(y_pred), y, weights='quadratic'), device='cuda:0' )<load_pretrained>
reg.score(X_train,y_train )
learn = Learner(data,
md_ef,
metrics = [qk],
model_dir="models" ).to_fp16()
learn.data.add_test(ImageList.from_df(test_df,
'.. /input/aptos2019-blindness-detection',
folder='test_images',
suffix='.png')) 
<load_pretrained>
test_data =pd.read_csv(at4_test_path,sep="," )
learn.load('abcdef');<compute_train_metric>
test_copy=test_data.copy()
test_min_max_scaler_spike = preprocessing.MinMaxScaler(feature_range=(0,1))
test_min_max_scaler_latency = preprocessing.MinMaxScaler(feature_range=(0,1))
test_min_max_scaler_rank = preprocessing.MinMaxScaler(feature_range=(0,1))
test_values_to10 = test_copy.iloc[:,0:10].values
test_values_to20 = test_copy.iloc[:,10:20].values
test_values_to30= test_copy.iloc[:,20:30].values
test_transformation_to10 = test_min_max_scaler_spike.fit(test_values_to10.T)
 test_values_scaled_10 = test_min_max_scaler_spike.transform(test_values_to10.T ).T
test_transformation_to20 = test_min_max_scaler_latency.fit(test_values_to20.T)
 test_values_scaled_20 = test_min_max_scaler_latency.transform(test_values_to20.T ).T
test_transformation_to30 = test_min_max_scaler_rank.fit(test_values_to30.T)
 test_values_scaled_30 = test_min_max_scaler_rank.transform(test_values_to30.T ).T
test_scaled=pd.DataFrame()
for i in range(0,10):
test_scaled['onsc'+str(i)]=pd.DataFrame(test_values_scaled_10 ).iloc[:,i]
for i in range(0,10):
test_scaled['onstf'+str(i)]=pd.DataFrame(test_values_scaled_20 ).iloc[:,i]
for i in range(0,10):
test_scaled['onsto'+str(i)]=pd.DataFrame(test_values_scaled_30 ).iloc[:,i]
class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
ll = metrics.cohen_kappa_score(y, X_p, weights='quadratic')
 return -ll
def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X=X, y=y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 print(-loss_partial(self.coef_['x']))
def predict(self, X, coef):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
return X_p
def coefficients(self):
return self.coef_['x']<save_to_csv>
X_test=calculate_score(test_scaled)
def run_subm(learn=learn, coefficients=[0.5, 1.5, 2.5, 3.5]):
opt = OptimizedRounder()
preds,y = learn.get_preds(DatasetType.Test)
 tst_pred = opt.predict(preds, coefficients)
 test_df.diagnosis = tst_pred.astype(int)
 test_df.to_csv('submission.csv',index=False)
 print('done' )<count_missing_values>
test_predictions=reg.predict(X_test)
 test_predictions
def missing_values_table(df):
mis_val = df.isnull().sum()
mis_val_percent = 100 * df.isnull().sum() / len(df)
 mis_val_table = pd.concat([mis_val, mis_val_percent], axis=1)
 mis_val_table_ren_columns = mis_val_table.rename(
 columns = {0 : 'Missing Values', 1 : '% of Total Values'})
 mis_val_table_ren_columns = mis_val_table_ren_columns[
mis_val_table_ren_columns.iloc[:,1] != 0].sort_values(
 '% of Total Values', ascending=False ).round(1)
 print("Your selected dataframe has " + str(df.shape[1])+ " columns.
 "
"There are " + str(mis_val_table_ren_columns.shape[0])+
" columns that have missing values.")
 return mis_val_table_ren_columns<drop_column>
l=len(test_predictions)
 result_array=[]
for i in range(0,l):
result_array.append([i,test_predictions[i]] )
missing_values = missing_values_table(traindata)
 missing_values
traindata[traindata['winPlacePerc'].isnull() ]
traindata.drop(2744604, inplace= True )<categorify>
results = pd.DataFrame(result_array,columns=["id","label"] )
<groupby><EOS> 
results.to_csv('submission.csv',index=False )