Kaggle competition: APTOS 2019 Blindness Detection
<SOS> metric: QuadraticWeightedKappa Kaggle data source: aptos-2019-blindness-detection<count_unique_values>
warnings.filterwarnings('ignore' )
print(f"Unique Countries: {len(df_train.Country_Region.unique())}")
<count_unique_values>
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.utils.model_zoo as model_zoo
import re
import math
from copy import deepcopy
print(f"Unique Regions: {df_train.shape[0]/len(df_train.Date.unique())}")
<count_values>
def load_pretrained(model, default_cfg, num_classes=1000, in_chans=3, filter_fn=None):
if 'url' not in default_cfg or not default_cfg['url']:
logging.warning("Pretrained model URL is invalid, using random initialization.")
 return
state_dict = model_zoo.load_url(default_cfg['url'])
 if in_chans == 1:
conv1_name = default_cfg['first_conv']
logging.info('Converting first conv(%s)from 3 to 1 channel' % conv1_name)
 conv1_weight = state_dict[conv1_name + '.weight']
state_dict[conv1_name + '.weight'] = conv1_weight.sum(dim=1, keepdim=True)
 elif in_chans != 3:
assert False, "Invalid in_chans for pretrained weights"
strict = True
classifier_name = default_cfg['classifier']
if num_classes == 1000 and default_cfg['num_classes'] == 1001:
classifier_weight = state_dict[classifier_name + '.weight']
state_dict[classifier_name + '.weight'] = classifier_weight[1:]
classifier_bias = state_dict[classifier_name + '.bias']
state_dict[classifier_name + '.bias'] = classifier_bias[1:]
elif num_classes != default_cfg['num_classes']:
del state_dict[classifier_name + '.weight']
del state_dict[classifier_name + '.bias']
strict = False
if filter_fn is not None:
state_dict = filter_fn(state_dict)
 model.load_state_dict(state_dict, strict=strict)
df_train.Country_Region.value_counts() 
<count_missing_values>
def adaptive_pool_feat_mult(pool_type='avg'):
if pool_type == 'catavgmax':
return 2
else:
return 1
def adaptive_avgmax_pool2d(x, output_size=1):
x_avg = F.adaptive_avg_pool2d(x, output_size)
 x_max = F.adaptive_max_pool2d(x, output_size)
 return 0.5 *(x_avg + x_max)
 def adaptive_catavgmax_pool2d(x, output_size=1):
x_avg = F.adaptive_avg_pool2d(x, output_size)
 x_max = F.adaptive_max_pool2d(x, output_size)
 return torch.cat(( x_avg, x_max), 1)
 def select_adaptive_pool2d(x, pool_type='avg', output_size=1):

 if pool_type == 'avg':
x = F.adaptive_avg_pool2d(x, output_size)
 elif pool_type == 'avgmax':
x = adaptive_avgmax_pool2d(x, output_size)
 elif pool_type == 'catavgmax':
x = adaptive_catavgmax_pool2d(x, output_size)
 elif pool_type == 'max':
x = F.adaptive_max_pool2d(x, output_size)
 else:
assert False, 'Invalid pool type: %s' % pool_type
return x
class AdaptiveAvgMaxPool2d(nn.Module):
def __init__(self, output_size=1):
super(AdaptiveAvgMaxPool2d, self ).__init__()
self.output_size = output_size
def forward(self, x):
return adaptive_avgmax_pool2d(x, self.output_size)
 class AdaptiveCatAvgMaxPool2d(nn.Module):
def __init__(self, output_size=1):
super(AdaptiveCatAvgMaxPool2d, self ).__init__()
self.output_size = output_size
def forward(self, x):
return adaptive_catavgmax_pool2d(x, self.output_size)
 class SelectAdaptivePool2d(nn.Module):

 def __init__(self, output_size=1, pool_type='avg'):
super(SelectAdaptivePool2d, self ).__init__()
self.output_size = output_size
self.pool_type = pool_type
if pool_type == 'avgmax':
self.pool = AdaptiveAvgMaxPool2d(output_size)
 elif pool_type == 'catavgmax':
self.pool = AdaptiveCatAvgMaxPool2d(output_size)
 elif pool_type == 'max':
self.pool = nn.AdaptiveMaxPool2d(output_size)
 else:
if pool_type != 'avg':
assert False, 'Invalid pool type: %s' % pool_type
self.pool = nn.AdaptiveAvgPool2d(output_size)
 def forward(self, x):
return self.pool(x)
 def feat_mult(self):
return adaptive_pool_feat_mult(self.pool_type)
 def __repr__(self):
return self.__class__.__name__ + '(' \
+ 'output_size=' + str(self.output_size)\
+ ', pool_type=' + self.pool_type + ')'
print(f"Number of rows without Country_Region : {df_train.Country_Region.isna().sum() }")
<feature_engineering>
IMAGENET_DEFAULT_MEAN =(0.485, 0.456, 0.406)
 IMAGENET_DEFAULT_STD =(0.229, 0.224, 0.225)
 def _cfg(url='', **kwargs):
return {
'url': url, 'num_classes': 1000, 'input_size':(3, 224, 224), 'pool_size':(7, 7),
'crop_pct': 0.875, 'interpolation': 'bicubic',
'mean': IMAGENET_DEFAULT_MEAN, 'std': IMAGENET_DEFAULT_STD,
'first_conv': 'conv_stem', 'classifier': 'classifier',
**kwargs
}
def _is_static_pad(kernel_size, stride=1, dilation=1, **_):
return stride == 1 and(dilation *(kernel_size - 1)) % 2 == 0
def _get_padding(kernel_size, stride=1, dilation=1, **_):
padding =(( stride - 1)+ dilation *(kernel_size - 1)) // 2
return padding
def _calc_same_pad(i, k, s, d):
return max(( math.ceil(i / s)- 1)* s +(k - 1)* d + 1 - i, 0)
 def _split_channels(num_chan, num_groups):
split = [num_chan // num_groups for _ in range(num_groups)]
split[0] += num_chan - sum(split)
 return split
class Conv2dSame(nn.Conv2d):

 def __init__(self, in_channels, out_channels, kernel_size, stride=1,
padding=0, dilation=1, groups=1, bias=True):
super(Conv2dSame, self ).__init__(
 in_channels, out_channels, kernel_size, stride, 0, dilation,
groups, bias)
 def forward(self, x):
ih, iw = x.size() [-2:]
kh, kw = self.weight.size() [-2:]
pad_h = _calc_same_pad(ih, kh, self.stride[0], self.dilation[0])
 pad_w = _calc_same_pad(iw, kw, self.stride[1], self.dilation[1])
 if pad_h > 0 or pad_w > 0:
x = F.pad(x, [pad_w//2, pad_w - pad_w//2, pad_h//2, pad_h - pad_h//2])
 return F.conv2d(x, self.weight, self.bias, self.stride,
self.padding, self.dilation, self.groups)
 def conv2d_pad(in_chs, out_chs, kernel_size, **kwargs):
padding = kwargs.pop('padding', '')
 kwargs.setdefault('bias', False)
 if isinstance(padding, str):
padding = padding.lower()
if padding == 'same':
if _is_static_pad(kernel_size, **kwargs):
padding = _get_padding(kernel_size, **kwargs)
 return nn.Conv2d(in_chs, out_chs, kernel_size, padding=padding, **kwargs)
 else:
return Conv2dSame(in_chs, out_chs, kernel_size, **kwargs)
 elif padding == 'valid':
return nn.Conv2d(in_chs, out_chs, kernel_size, padding=0, **kwargs)
 else:
padding = _get_padding(kernel_size, **kwargs)
 return nn.Conv2d(in_chs, out_chs, kernel_size, padding=padding, **kwargs)
 else:
return nn.Conv2d(in_chs, out_chs, kernel_size, padding=padding, **kwargs)
 def select_conv2d(in_chs, out_chs, kernel_size, **kwargs):
assert 'groups' not in kwargs
if isinstance(kernel_size, list):
return MixedConv2d(in_chs, out_chs, kernel_size, **kwargs)
 else:
depthwise = kwargs.pop('depthwise', False)
 groups = out_chs if depthwise else 1
return conv2d_pad(in_chs, out_chs, kernel_size, groups=groups, **kwargs)
 def _parse_ksize(ss):
if ss.isdigit() :
return int(ss)
 else:
return [int(k)for k in ss.split('.')]
def _resolve_bn_args(kwargs):
bn_args = _BN_ARGS_TF.copy() if kwargs.pop('bn_tf', False)else _BN_ARGS_PT.copy()
bn_momentum = kwargs.pop('bn_momentum', None)
 if bn_momentum is not None:
bn_args['momentum'] = bn_momentum
bn_eps = kwargs.pop('bn_eps', None)
 if bn_eps is not None:
bn_args['eps'] = bn_eps
return bn_args
def _round_channels(channels, multiplier=1.0, divisor=8, channel_min=None):

 if not multiplier:
return channels
channels *= multiplier
channel_min = channel_min or divisor
new_channels = max(
 int(channels + divisor / 2)// divisor * divisor,
channel_min)
 if new_channels < 0.9 * channels:
new_channels += divisor
return new_channels
def _decode_arch_def(arch_def, depth_multiplier=1.0):
arch_args = []
for stack_idx, block_strings in enumerate(arch_def):
assert isinstance(block_strings, list)
 stack_args = []
for block_str in block_strings:
assert isinstance(block_str, str)
 stack_args.extend(_decode_block_str(block_str, depth_multiplier))
arch_args.append(stack_args)
 return arch_args
def _decode_block_str(block_str, depth_multiplier=1.0):

 assert isinstance(block_str, str)
 ops = block_str.split('_')
 block_type = ops[0]
ops = ops[1:]
options = {}
noskip = False
for op in ops:
if op == 'noskip':
noskip = True
elif op.startswith('n'):
key = op[0]
v = op[1:]
if v == 're':
value = F.relu
elif v == 'r6':
value = F.relu6
elif v == 'hs':
value = hard_swish
elif v == 'sw':
value = swish
else:
continue
options[key] = value
else:
splits = re.split(r'(\d.*)', op)
 if len(splits)>= 2:
key, value = splits[:2]
options[key] = value
act_fn = options['n'] if 'n' in options else None
exp_kernel_size = _parse_ksize(options['a'])if 'a' in options else 1
pw_kernel_size = _parse_ksize(options['p'])if 'p' in options else 1
num_repeat = int(options['r'])
 if block_type == 'ir':
block_args = dict(
 block_type=block_type,
dw_kernel_size=_parse_ksize(options['k']),
exp_kernel_size=exp_kernel_size,
pw_kernel_size=pw_kernel_size,
out_chs=int(options['c']),
exp_ratio=float(options['e']),
se_ratio=float(options['se'])if 'se' in options else None,
stride=int(options['s']),
act_fn=act_fn,
noskip=noskip, 
)
 elif block_type == 'ds' or block_type == 'dsa':
block_args = dict(
 block_type=block_type,
dw_kernel_size=_parse_ksize(options['k']),
pw_kernel_size=pw_kernel_size,
out_chs=int(options['c']),
se_ratio=float(options['se'])if 'se' in options else None,
stride=int(options['s']),
act_fn=act_fn,
pw_act=block_type == 'dsa',
noskip=block_type == 'dsa' or noskip, 
)
 elif block_type == 'cn':
block_args = dict(
 block_type=block_type,
kernel_size=int(options['k']),
out_chs=int(options['c']),
stride=int(options['s']),
act_fn=act_fn, 
)
 else:
assert False, 'Unknown block type(%s)' % block_type
num_repeat = int(math.ceil(num_repeat * depth_multiplier))
return [deepcopy(block_args)for _ in range(num_repeat)]
default_cfgs = {
'efficientnet_b0': _cfg(
 url='https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/efficientnet_b0-d6904d92.pth'),
'efficientnet_b1': _cfg(
 url='https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/efficientnet_b1-533bc792.pth',
input_size=(3, 240, 240), pool_size=(8, 8), crop_pct=0.882),
'efficientnet_b2': _cfg(
 url='https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/efficientnet_b2-cf78dc4d.pth',
input_size=(3, 260, 260), pool_size=(9, 9), crop_pct=0.890),
'efficientnet_b3': _cfg(
 url='', input_size=(3, 300, 300), pool_size=(10, 10), crop_pct=0.904),
'efficientnet_b4': _cfg(
 url='', input_size=(3, 380, 380), pool_size=(12, 12), crop_pct=0.922),
'efficientnet_b5': _cfg(
 url='', input_size=(3, 456, 456), pool_size=(15, 15), crop_pct=0.934),
'efficientnet_b6': _cfg(
 url='', input_size=(3, 528, 528), pool_size=(17, 17), crop_pct=0.942),
'efficientnet_b7': _cfg(
 url='', input_size=(3, 600, 600), pool_size=(19, 19), crop_pct=0.949),
'tf_efficientnet_b0': _cfg(
 url='https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/tf_efficientnet_b0_aa-827b6e33.pth',
input_size=(3, 224, 224)) ,
'tf_efficientnet_b1': _cfg(
 url='https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/tf_efficientnet_b1_aa-ea7a6ee0.pth',
input_size=(3, 240, 240), pool_size=(8, 8), crop_pct=0.882),
'tf_efficientnet_b2': _cfg(
 url='https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/tf_efficientnet_b2_aa-60c94f97.pth',
input_size=(3, 260, 260), pool_size=(9, 9), crop_pct=0.890),
'tf_efficientnet_b3': _cfg(
 url='https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/tf_efficientnet_b3_aa-84b4657e.pth',
input_size=(3, 300, 300), pool_size=(10, 10), crop_pct=0.904),
'tf_efficientnet_b4': _cfg(
 url='https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/tf_efficientnet_b4_aa-818f208c.pth',
input_size=(3, 380, 380), pool_size=(12, 12), crop_pct=0.922),
'tf_efficientnet_b5': _cfg(
 url='https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/tf_efficientnet_b5_aa-99018a74.pth',
input_size=(3, 456, 456), pool_size=(15, 15), crop_pct=0.934),
'tf_efficientnet_b6': _cfg(
 url='https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/tf_efficientnet_b6_aa-80ba17e4.pth',
input_size=(3, 528, 528), pool_size=(17, 17), crop_pct=0.942),
'tf_efficientnet_b7': _cfg(
 url='https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/tf_efficientnet_b7_aa-076e3472.pth',
input_size=(3, 600, 600), pool_size=(19, 19), crop_pct=0.949),
}
_DEBUG = False
_BN_MOMENTUM_PT_DEFAULT = 0.1
_BN_EPS_PT_DEFAULT = 1e-5
_BN_ARGS_PT = dict(momentum=_BN_MOMENTUM_PT_DEFAULT, eps=_BN_EPS_PT_DEFAULT)
 _BN_MOMENTUM_TF_DEFAULT = 1 - 0.99
_BN_EPS_TF_DEFAULT = 1e-3
_BN_ARGS_TF = dict(momentum=_BN_MOMENTUM_TF_DEFAULT, eps=_BN_EPS_TF_DEFAULT)
 def _initialize_weight_goog(m):
if isinstance(m, nn.Conv2d):
n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
m.weight.data.normal_(0, math.sqrt(2.0 / n))
if m.bias is not None:
m.bias.data.zero_()
elif isinstance(m, nn.BatchNorm2d):
m.weight.data.fill_(1.0)
 m.bias.data.zero_()
elif isinstance(m, nn.Linear):
n = m.weight.size(0)
 init_range = 1.0 / math.sqrt(n)
 m.weight.data.uniform_(-init_range, init_range)
 m.bias.data.zero_()
def swish(x, inplace=False):
if inplace:
return x.mul_(x.sigmoid())
 else:
return x * x.sigmoid()
def sigmoid(x, inplace=False):
return x.sigmoid_() if inplace else x.sigmoid()
class InvertedResidual(nn.Module):

 def __init__(self, in_chs, out_chs, dw_kernel_size=3,
stride=1, pad_type='', act_fn=F.relu, noskip=False,
exp_ratio=1.0, exp_kernel_size=1, pw_kernel_size=1,
se_ratio=0., se_reduce_mid=False, se_gate_fn=sigmoid,
shuffle_type=None, bn_args=_BN_ARGS_PT, drop_connect_rate=0.) :
super(InvertedResidual, self ).__init__()
mid_chs = int(in_chs * exp_ratio)
 self.has_se = se_ratio is not None and se_ratio > 0.
 self.has_residual =(in_chs == out_chs and stride == 1)and not noskip
self.act_fn = act_fn
self.drop_connect_rate = drop_connect_rate
self.conv_pw = select_conv2d(in_chs, mid_chs, exp_kernel_size, padding=pad_type)
 self.bn1 = nn.BatchNorm2d(mid_chs, **bn_args)
 self.shuffle_type = shuffle_type
if shuffle_type is not None and isinstance(exp_kernel_size, list):
self.shuffle = ChannelShuffle(len(exp_kernel_size))
self.conv_dw = select_conv2d(
 mid_chs, mid_chs, dw_kernel_size, stride=stride, padding=pad_type, depthwise=True)
 self.bn2 = nn.BatchNorm2d(mid_chs, **bn_args)
 if self.has_se:
se_base_chs = mid_chs if se_reduce_mid else in_chs
self.se = SqueezeExcite(
 mid_chs, reduce_chs=max(1, int(se_base_chs * se_ratio)) , act_fn=act_fn, gate_fn=se_gate_fn)
 self.conv_pwl = select_conv2d(mid_chs, out_chs, pw_kernel_size, padding=pad_type)
 self.bn3 = nn.BatchNorm2d(out_chs, **bn_args)
 def forward(self, x):
residual = x
x = self.conv_pw(x)
 x = self.bn1(x)
 x = self.act_fn(x, inplace=True)
 if self.shuffle_type == "mid":
x = self.shuffle(x)
 x = self.conv_dw(x)
 x = self.bn2(x)
 x = self.act_fn(x, inplace=True)
 if self.has_se:
x = self.se(x)
 x = self.conv_pwl(x)
 x = self.bn3(x)
 if self.has_residual:
if self.drop_connect_rate > 0.:
x = drop_connect(x, self.training, self.drop_connect_rate)
 x += residual
return x
class GenEfficientNet(nn.Module):

 def __init__(self, block_args, num_classes=1000, in_chans=3, stem_size=32, num_features=1280,
channel_multiplier=1.0, channel_divisor=8, channel_min=None,
pad_type='', act_fn=F.relu, drop_rate=0., drop_connect_rate=0.,
se_gate_fn=sigmoid, se_reduce_mid=False, bn_args=_BN_ARGS_PT,
global_pool='avg', head_conv='default', weight_init='goog'):
super(GenEfficientNet, self ).__init__()
self.num_classes = num_classes
self.drop_rate = drop_rate
self.act_fn = act_fn
self.num_features = num_features
stem_size = _round_channels(stem_size, channel_multiplier, channel_divisor, channel_min)
 self.conv_stem = select_conv2d(in_chans, stem_size, 3, stride=2, padding=pad_type)
 self.bn1 = nn.BatchNorm2d(stem_size, **bn_args)
 in_chs = stem_size
builder = _BlockBuilder(
 channel_multiplier, channel_divisor, channel_min,
pad_type, act_fn, se_gate_fn, se_reduce_mid,
bn_args, drop_connect_rate, verbose=_DEBUG)
 self.blocks = nn.Sequential(*builder(in_chs, block_args))
in_chs = builder.in_chs
if not head_conv or head_conv == 'none':
self.efficient_head = False
self.conv_head = None
assert in_chs == self.num_features
else:
self.efficient_head = head_conv == 'efficient'
self.conv_head = select_conv2d(in_chs, self.num_features, 1, padding=pad_type)
 self.bn2 = None if self.efficient_head else nn.BatchNorm2d(self.num_features, **bn_args)
 self.global_pool = SelectAdaptivePool2d(pool_type=global_pool)
 self.classifier = nn.Linear(self.num_features * self.global_pool.feat_mult() , self.num_classes)
 for m in self.modules() :
if weight_init == 'goog':
_initialize_weight_goog(m)
 else:
_initialize_weight_default(m)
 def get_classifier(self):
return self.classifier
def reset_classifier(self, num_classes, global_pool='avg'):
self.global_pool = SelectAdaptivePool2d(pool_type=global_pool)
 self.num_classes = num_classes
del self.classifier
if num_classes:
self.classifier = nn.Linear(
 self.num_features * self.global_pool.feat_mult() , num_classes)
 else:
self.classifier = None
def forward_features(self, x, pool=True):
x = self.conv_stem(x)
 x = self.bn1(x)
 x = self.act_fn(x, inplace=True)
 x = self.blocks(x)
 if self.efficient_head:
x = self.global_pool(x)
 x = self.conv_head(x)
 x = self.act_fn(x, inplace=True)
 if pool:
x = x.view(x.size(0), -1)
 else:
if self.conv_head is not None:
x = self.conv_head(x)
 x = self.bn2(x)
 x = self.act_fn(x, inplace=True)
 if pool:
x = self.global_pool(x)
 x = x.view(x.size(0), -1)
 return x
def forward(self, x):
x = self.forward_features(x)
 if self.drop_rate > 0.:
x = F.dropout(x, p=self.drop_rate, training=self.training)
 return self.classifier(x)
 class DepthwiseSeparableConv(nn.Module):

 def __init__(self, in_chs, out_chs, dw_kernel_size=3,
stride=1, pad_type='', act_fn=F.relu, noskip=False,
pw_kernel_size=1, pw_act=False,
se_ratio=0., se_gate_fn=sigmoid,
bn_args=_BN_ARGS_PT, drop_connect_rate=0.) :
super(DepthwiseSeparableConv, self ).__init__()
assert stride in [1, 2]
self.has_se = se_ratio is not None and se_ratio > 0.
 self.has_residual =(stride == 1 and in_chs == out_chs)and not noskip
self.has_pw_act = pw_act
self.act_fn = act_fn
self.drop_connect_rate = drop_connect_rate
self.conv_dw = select_conv2d(
 in_chs, in_chs, dw_kernel_size, stride=stride, padding=pad_type, depthwise=True)
 self.bn1 = nn.BatchNorm2d(in_chs, **bn_args)
 if self.has_se:
self.se = SqueezeExcite(
 in_chs, reduce_chs=max(1, int(in_chs * se_ratio)) , act_fn=act_fn, gate_fn=se_gate_fn)
 self.conv_pw = select_conv2d(in_chs, out_chs, pw_kernel_size, padding=pad_type)
 self.bn2 = nn.BatchNorm2d(out_chs, **bn_args)
 def forward(self, x):
residual = x
x = self.conv_dw(x)
 x = self.bn1(x)
 x = self.act_fn(x, inplace=True)
 if self.has_se:
x = self.se(x)
 x = self.conv_pw(x)
 x = self.bn2(x)
 if self.has_pw_act:
x = self.act_fn(x, inplace=True)
 if self.has_residual:
if self.drop_connect_rate > 0.:
x = drop_connect(x, self.training, self.drop_connect_rate)
 x += residual
return x
class SqueezeExcite(nn.Module):
def __init__(self, in_chs, reduce_chs=None, act_fn=F.relu, gate_fn=sigmoid):
super(SqueezeExcite, self ).__init__()
self.act_fn = act_fn
self.gate_fn = gate_fn
reduced_chs = reduce_chs or in_chs
self.conv_reduce = nn.Conv2d(in_chs, reduced_chs, 1, bias=True)
 self.conv_expand = nn.Conv2d(reduced_chs, in_chs, 1, bias=True)
 def forward(self, x):
x_se = x.view(x.size(0), x.size(1), -1 ).mean(-1 ).view(x.size(0), x.size(1), 1, 1)
 x_se = self.conv_reduce(x_se)
 x_se = self.act_fn(x_se, inplace=True)
 x_se = self.conv_expand(x_se)
 x = x * self.gate_fn(x_se)
 return x
class _BlockBuilder:

 def __init__(self, channel_multiplier=1.0, channel_divisor=8, channel_min=None,
pad_type='', act_fn=None, se_gate_fn=sigmoid, se_reduce_mid=False,
bn_args=_BN_ARGS_PT, drop_connect_rate=0., verbose=False):
self.channel_multiplier = channel_multiplier
self.channel_divisor = channel_divisor
self.channel_min = channel_min
self.pad_type = pad_type
self.act_fn = act_fn
self.se_gate_fn = se_gate_fn
self.se_reduce_mid = se_reduce_mid
self.bn_args = bn_args
self.drop_connect_rate = drop_connect_rate
self.verbose = verbose
self.in_chs = None
self.block_idx = 0
self.block_count = 0
def _round_channels(self, chs):
return _round_channels(chs, self.channel_multiplier, self.channel_divisor, self.channel_min)
 def _make_block(self, ba):
bt = ba.pop('block_type')
 ba['in_chs'] = self.in_chs
ba['out_chs'] = self._round_channels(ba['out_chs'])
 ba['bn_args'] = self.bn_args
ba['pad_type'] = self.pad_type
ba['act_fn'] = ba['act_fn'] if ba['act_fn'] is not None else self.act_fn
assert ba['act_fn'] is not None
if bt == 'ir':
ba['drop_connect_rate'] = self.drop_connect_rate * self.block_idx / self.block_count
ba['se_gate_fn'] = self.se_gate_fn
ba['se_reduce_mid'] = self.se_reduce_mid
if self.verbose:
logging.info(' InvertedResidual {}, Args: {}'.format(self.block_idx, str(ba)))
 block = InvertedResidual(**ba)
 elif bt == 'ds' or bt == 'dsa':
ba['drop_connect_rate'] = self.drop_connect_rate * self.block_idx / self.block_count
if self.verbose:
logging.info(' DepthwiseSeparable {}, Args: {}'.format(self.block_idx, str(ba)))
 block = DepthwiseSeparableConv(**ba)
 elif bt == 'cn':
if self.verbose:
logging.info(' ConvBnAct {}, Args: {}'.format(self.block_idx, str(ba)))
 block = ConvBnAct(**ba)
 else:
assert False, 'Uknkown block type(%s)while building model.' % bt
self.in_chs = ba['out_chs']
return block
def _make_stack(self, stack_args):
blocks = []
for i, ba in enumerate(stack_args):
if self.verbose:
logging.info(' Block: {}'.format(i))
if i >= 1:
ba['stride'] = 1
block = self._make_block(ba)
 blocks.append(block)
 self.block_idx += 1
return nn.Sequential(*blocks)
 def __call__(self, in_chs, block_args):

 if self.verbose:
logging.info('Building model trunk with %d stages...' % len(block_args))
self.in_chs = in_chs
self.block_count = sum([len(x)for x in block_args])
 self.block_idx = 0
blocks = []
for stack_idx, stack in enumerate(block_args):
if self.verbose:
logging.info('Stack: {}'.format(stack_idx))
assert isinstance(stack, list)
 stack = self._make_stack(stack)
 blocks.append(stack)
 return blocks
def _gen_efficientnet(channel_multiplier=1.0, depth_multiplier=1.0, num_classes=1000, **kwargs):

 arch_def = [
['ds_r1_k3_s1_e1_c16_se0.25'],
['ir_r2_k3_s2_e6_c24_se0.25'],
['ir_r2_k5_s2_e6_c40_se0.25'],
['ir_r3_k3_s2_e6_c80_se0.25'],
['ir_r3_k5_s1_e6_c112_se0.25'],
['ir_r4_k5_s2_e6_c192_se0.25'],
['ir_r1_k3_s1_e6_c320_se0.25'],
]
num_features = _round_channels(1280, channel_multiplier, 8, None)
 model = GenEfficientNet(
 _decode_arch_def(arch_def, depth_multiplier),
num_classes=num_classes,
stem_size=32,
channel_multiplier=channel_multiplier,
num_features=num_features,
bn_args=_resolve_bn_args(kwargs),
act_fn=swish,
**kwargs 
)
 return model
def tf_efficientnet_b0(pretrained=False, num_classes=1000, in_chans=3, **kwargs):

 default_cfg = default_cfgs['tf_efficientnet_b0']
kwargs['bn_eps'] = _BN_EPS_TF_DEFAULT
kwargs['pad_type'] = 'same'
model = _gen_efficientnet(
 channel_multiplier=1.0, depth_multiplier=1.0,
num_classes=num_classes, in_chans=in_chans, **kwargs)
 model.default_cfg = default_cfg
if pretrained:
load_pretrained(model, default_cfg, num_classes, in_chans)
 return model
def tf_efficientnet_b1(pretrained=False, num_classes=1000, in_chans=3, **kwargs):

 default_cfg = default_cfgs['tf_efficientnet_b1']
kwargs['bn_eps'] = _BN_EPS_TF_DEFAULT
kwargs['pad_type'] = 'same'
model = _gen_efficientnet(
 channel_multiplier=1.0, depth_multiplier=1.1,
num_classes=num_classes, in_chans=in_chans, **kwargs)
 model.default_cfg = default_cfg
if pretrained:
load_pretrained(model, default_cfg, num_classes, in_chans)
 return model
def tf_efficientnet_b2(pretrained=False, num_classes=1000, in_chans=3, **kwargs):

 default_cfg = default_cfgs['tf_efficientnet_b2']
kwargs['bn_eps'] = _BN_EPS_TF_DEFAULT
kwargs['pad_type'] = 'same'
model = _gen_efficientnet(
 channel_multiplier=1.1, depth_multiplier=1.2,
num_classes=num_classes, in_chans=in_chans, **kwargs)
 model.default_cfg = default_cfg
if pretrained:
load_pretrained(model, default_cfg, num_classes, in_chans)
 return model
def tf_efficientnet_b3(pretrained=False, num_classes=1000, in_chans=3, **kwargs):

 default_cfg = default_cfgs['tf_efficientnet_b3']
kwargs['bn_eps'] = _BN_EPS_TF_DEFAULT
kwargs['pad_type'] = 'same'
model = _gen_efficientnet(
 channel_multiplier=1.2, depth_multiplier=1.4,
num_classes=num_classes, in_chans=in_chans, **kwargs)
 model.default_cfg = default_cfg
if pretrained:
load_pretrained(model, default_cfg, num_classes, in_chans)
 return model
def tf_efficientnet_b4(pretrained=False, num_classes=1000, in_chans=3, **kwargs):

 default_cfg = default_cfgs['tf_efficientnet_b4']
kwargs['bn_eps'] = _BN_EPS_TF_DEFAULT
kwargs['pad_type'] = 'same'
model = _gen_efficientnet(
 channel_multiplier=1.4, depth_multiplier=1.8,
num_classes=num_classes, in_chans=in_chans, **kwargs)
 model.default_cfg = default_cfg
if pretrained:
load_pretrained(model, default_cfg, num_classes, in_chans)
 return model
def tf_efficientnet_b5(pretrained=False, num_classes=1000, in_chans=3, **kwargs):

 default_cfg = default_cfgs['tf_efficientnet_b5']
kwargs['bn_eps'] = _BN_EPS_TF_DEFAULT
kwargs['pad_type'] = 'same'
model = _gen_efficientnet(
 channel_multiplier=1.6, depth_multiplier=2.2,
num_classes=num_classes, in_chans=in_chans, **kwargs)
 model.default_cfg = default_cfg
if pretrained:
load_pretrained(model, default_cfg, num_classes, in_chans)
 return model
def tf_efficientnet_b6(pretrained=False, num_classes=1000, in_chans=3, **kwargs):

 default_cfg = default_cfgs['tf_efficientnet_b6']
kwargs['bn_eps'] = _BN_EPS_TF_DEFAULT
kwargs['pad_type'] = 'same'
model = _gen_efficientnet(
 channel_multiplier=1.8, depth_multiplier=2.6,
num_classes=num_classes, in_chans=in_chans, **kwargs)
 model.default_cfg = default_cfg
if pretrained:
load_pretrained(model, default_cfg, num_classes, in_chans)
 return model
def tf_efficientnet_b7(pretrained=False, num_classes=1000, in_chans=3, **kwargs):

 default_cfg = default_cfgs['tf_efficientnet_b7']
kwargs['bn_eps'] = _BN_EPS_TF_DEFAULT
kwargs['pad_type'] = 'same'
model = _gen_efficientnet(
 channel_multiplier=2.0, depth_multiplier=3.1,
num_classes=num_classes, in_chans=in_chans, **kwargs)
 model.default_cfg = default_cfg
if pretrained:
load_pretrained(model, default_cfg, num_classes, in_chans)
 return model
df_train["UniqueRegion"]=df_train.Country_Region
df_train.UniqueRegion[df_train.Province_State.isna() ==False]=df_train.Province_State+" , "+df_train.Country_Region
df_train[df_train.Province_State.isna() ==False] 
<drop_column>
from fastai import *
from fastai.vision import *
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
import os
import scipy as sp
from functools import partial
from sklearn import metrics
from collections import Counter
from fastai.callbacks import *
import PIL
import cv2
df_train.drop(labels=["Id","Province_State","Country_Region"], axis=1, inplace=True)
<feature_engineering>
def seed_everything(seed=77):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything()
test_dates=list(df_test.Date.unique())
 print(f"Period :{len(df_test.Date.unique())} days")
 print(f"From : {df_test.Date.min() } To : {df_test.Date.max() }" )<define_variables>
LOCAL = False
print(f"Total Regions : {df_test.shape[0]/43}" )<feature_engineering>
def get_df() :
if LOCAL:
base_image_dir = os.path.join('.', 'data/')
 else:
base_image_dir = os.path.join('.. ', 'input/aptos2019-blindness-detection/')
 train_dir = os.path.join(base_image_dir,'train_images/')
 df = pd.read_csv(os.path.join(base_image_dir, 'train.csv'))
df['path'] = df['id_code'].map(lambda x: os.path.join(train_dir,'{}.png'.format(x)))
 df = df.drop(columns=['id_code'])
 df = df.sample(frac=1 ).reset_index(drop=True)
 test_df = pd.read_csv(os.path.join(base_image_dir, 'sample_submission.csv'))
return df, test_df
df, test_df = get_df()
df_test["UniqueRegion"]=df_test.Country_Region
df_test.UniqueRegion[df_test.Province_State.isna() ==False]=df_test.Province_State+" , "+df_test.Country_Region
df_test.drop(labels=["Province_State","Country_Region"], axis=1, inplace=True )<count_unique_values>
def qk(y_pred, y):
return torch.tensor(cohen_kappa_score(torch.round(y_pred), y,
weights='quadratic'), device='cuda:0' )
len(df_test.UniqueRegion.unique() )<define_variables>
!mkdir models
!cp '.. /input/effinet-b5-aptos/Effi0_12e_pretrained.pth' "models/"
!cp '.. /input/effinet-b5-aptos/Effi1_12e_pretrained.pth' "models/"
!cp '.. /input/effinet-b5-aptos/Effi2_10e_pretrained.pth' "models/"
!cp '.. /input/effinet-b5-aptos/Effi3_10e_pretrained.pth' "models/"
!cp '.. /input/effinet-b5-aptos/best_effi6_224.pth' "models/"
!cp '.. /input/starter-kernel-for-0-79/models/abcdef.pth' "models/"
only_train_dates=set(train_dates)-set(test_dates)
 print("Only train dates : ",len(only_train_dates))
intersection_dates=set(test_dates)&set(train_dates)
 print("Intersection dates : ",len(intersection_dates))
only_test_dates=set(test_dates)-set(train_dates)
 print("Only Test dates : ",len(only_test_dates))<feature_engineering>
data2 =(ImageList.from_df(df=df,path='./',cols='path')
.split_by_rand_pct(0.2)
.label_from_df(cols='diagnosis',label_cls=FloatList)
.transform(tfms,size=sz,resize_method=ResizeMethod.SQUISH,padding_mode='zeros')
.databunch(bs=bs)
.normalize(imagenet_stats)
)
 if not LOCAL:
data2.add_test(ImageList.from_df(test_df, '.. /input/aptos2019-blindness-detection/',
folder='test_images',
suffix='.png'))
bs = 32
sz = 224
tfms = get_transforms(do_flip=True,flip_vert=True)
 learn2 = Learner(data2,
tf_efficientnet_b2(pretrained=False, num_classes=1),
metrics = [qk],
model_dir="models" ).to_fp16()
learn2.load('Effi2_10e_pretrained');
preds2,y = learn2.get_preds(DatasetType.Test)
 del learn2, data2
df_test_temp=pd.DataFrame()
df_test_temp["Date"]=df_test.Date
df_test_temp["ConfirmedCases"]=0.0
df_test_temp["Fatalities"]=0.0
df_test_temp["UniqueRegion"]=df_test.UniqueRegion
df_test_temp["Delta"]=1.0<feature_engineering>
gc.collect() ;
%%time
final_df=pd.DataFrame(columns=["Date","ConfirmedCases","Fatalities","UniqueRegion"])
 for region in df_train.UniqueRegion.unique() :
df_temp=df_train[df_train.UniqueRegion==region].reset_index()
df_temp["Delta"]=1.0
size_train=df_temp.shape[0]
for i in range(1,df_temp.shape[0]):
if(df_temp.ConfirmedCases[i-1]>0):
df_temp.Delta[i]=df_temp.ConfirmedCases[i]/df_temp.ConfirmedCases[i-1]
n=5
delta_list=df_temp.tail(n ).Delta
delta_avg=df_temp.tail(n ).Delta.mean()
death_rate=df_temp.tail(1 ).Fatalities.sum() /df_temp.tail(1 ).ConfirmedCases.sum()
df_test_app=df_test_temp[df_test_temp.UniqueRegion==region]
df_test_app=df_test_app[df_test_app.Date>df_temp.Date.max() ]
X=np.arange(1,n+1 ).reshape(-1,1)
 Y=delta_list
model=LinearRegression()
model.fit(X,Y)
 df_temp=pd.concat([df_temp,df_test_app])
 df_temp=df_temp.reset_index()
for i in range(size_train, df_temp.shape[0]):
n=n+1
df_temp.Delta[i]=max(1,model.predict(np.array([n] ).reshape(-1,1)) [0])
 for i in range(size_train, df_temp.shape[0]):
df_temp.ConfirmedCases[i]=round(df_temp.ConfirmedCases[i-1]*df_temp.Delta[i],0)
 df_temp.Fatalities[i]=round(death_rate*df_temp.ConfirmedCases[i],0)
 size_test=df_temp.shape[0]-df_test_temp[df_test_temp.UniqueRegion==region].shape[0]
df_temp=df_temp.iloc[size_test:,:]
df_temp=df_temp[["Date","ConfirmedCases","Fatalities","UniqueRegion","Delta"]]
final_df=pd.concat([final_df,df_temp], ignore_index=True)
 final_df.shape<save_to_csv>
gc.collect() ;
df_sub.Fatalities=final_df.Fatalities
df_sub.ConfirmedCases=final_df.ConfirmedCases
df_sub.to_csv("submission.csv", index=None )<load_from_csv>
GlobalParams = collections.namedtuple('GlobalParams', [
'batch_norm_momentum', 'batch_norm_epsilon', 'dropout_rate',
'num_classes', 'width_coefficient', 'depth_coefficient',
'depth_divisor', 'min_depth', 'drop_connect_rate', 'image_size'])
 BlockArgs = collections.namedtuple('BlockArgs', [
'kernel_size', 'num_repeat', 'input_filters', 'output_filters',
'expand_ratio', 'id_skip', 'stride', 'se_ratio'])
 GlobalParams.__new__.__defaults__ =(None,)* len(GlobalParams._fields)
 BlockArgs.__new__.__defaults__ =(None,)* len(BlockArgs._fields)
 def relu_fn(x):

 return x * torch.sigmoid(x)
 def round_filters(filters, global_params):

 multiplier = global_params.width_coefficient
if not multiplier:
return filters
divisor = global_params.depth_divisor
min_depth = global_params.min_depth
filters *= multiplier
min_depth = min_depth or divisor
new_filters = max(min_depth, int(filters + divisor / 2)// divisor * divisor)
 if new_filters < 0.9 * filters:
new_filters += divisor
return int(new_filters)
 def round_repeats(repeats, global_params):

 multiplier = global_params.depth_coefficient
if not multiplier:
return repeats
return int(math.ceil(multiplier * repeats))
def drop_connect(inputs, p, training):

 if not training: return inputs
batch_size = inputs.shape[0]
keep_prob = 1 - p
random_tensor = keep_prob
random_tensor += torch.rand([batch_size, 1, 1, 1], dtype=inputs.dtype, device=inputs.device)
 binary_tensor = torch.floor(random_tensor)
 output = inputs / keep_prob * binary_tensor
return output
def get_same_padding_conv2d(image_size=None):

 if image_size is None:
return Conv2dDynamicSamePadding
else:
return partial(Conv2dStaticSamePadding, image_size=image_size)
 class Conv2dDynamicSamePadding(nn.Conv2d):

 def __init__(self, in_channels, out_channels, kernel_size, stride=1, dilation=1, groups=1, bias=True):
super().__init__(in_channels, out_channels, kernel_size, stride, 0, dilation, groups, bias)
 self.stride = self.stride if len(self.stride)== 2 else [self.stride[0]]*2
def forward(self, x):
ih, iw = x.size() [-2:]
kh, kw = self.weight.size() [-2:]
sh, sw = self.stride
oh, ow = math.ceil(ih / sh), math.ceil(iw / sw)
 pad_h = max(( oh - 1)* self.stride[0] +(kh - 1)* self.dilation[0] + 1 - ih, 0)
 pad_w = max(( ow - 1)* self.stride[1] +(kw - 1)* self.dilation[1] + 1 - iw, 0)
 if pad_h > 0 or pad_w > 0:
x = F.pad(x, [pad_w//2, pad_w - pad_w//2, pad_h//2, pad_h - pad_h//2])
 return F.conv2d(x, self.weight, self.bias, self.stride, self.padding, self.dilation, self.groups)
 class Conv2dStaticSamePadding(nn.Conv2d):

 def __init__(self, in_channels, out_channels, kernel_size, image_size=None, **kwargs):
super().__init__(in_channels, out_channels, kernel_size, **kwargs)
 self.stride = self.stride if len(self.stride)== 2 else [self.stride[0]] * 2
assert image_size is not None
ih, iw = image_size if type(image_size)== list else [image_size, image_size]
kh, kw = self.weight.size() [-2:]
sh, sw = self.stride
oh, ow = math.ceil(ih / sh), math.ceil(iw / sw)
 pad_h = max(( oh - 1)* self.stride[0] +(kh - 1)* self.dilation[0] + 1 - ih, 0)
 pad_w = max(( ow - 1)* self.stride[1] +(kw - 1)* self.dilation[1] + 1 - iw, 0)
 if pad_h > 0 or pad_w > 0:
self.static_padding = nn.ZeroPad2d(( pad_w // 2, pad_w - pad_w // 2, pad_h // 2, pad_h - pad_h // 2))
else:
self.static_padding = Identity()
def forward(self, x):
x = self.static_padding(x)
 x = F.conv2d(x, self.weight, self.bias, self.stride, self.padding, self.dilation, self.groups)
 return x
class Identity(nn.Module):
def __init__(self,):
super(Identity, self ).__init__()
def forward(self, input):
return input
def efficientnet_params(model_name):

 params_dict = {
'efficientnet-b0':(1.0, 1.0, 224, 0.2),
'efficientnet-b1':(1.0, 1.1, 240, 0.2),
'efficientnet-b2':(1.1, 1.2, 260, 0.3),
'efficientnet-b3':(1.2, 1.4, 300, 0.3),
'efficientnet-b4':(1.4, 1.8, 380, 0.4),
'efficientnet-b5':(1.6, 2.2, 456, 0.4),
'efficientnet-b6':(1.8, 2.6, 528, 0.5),
'efficientnet-b7':(2.0, 3.1, 600, 0.5),
}
return params_dict[model_name]
class BlockDecoder(object):

 @staticmethod
def _decode_block_string(block_string):

 assert isinstance(block_string, str)
 ops = block_string.split('_')
 options = {}
for op in ops:
splits = re.split(r'(\d.*)', op)
 if len(splits)>= 2:
key, value = splits[:2]
options[key] = value
assert(( 's' in options and len(options['s'])== 1)or 
(len(options['s'])== 2 and options['s'][0] == options['s'][1]))
return BlockArgs(
 kernel_size=int(options['k']),
num_repeat=int(options['r']),
input_filters=int(options['i']),
output_filters=int(options['o']),
expand_ratio=int(options['e']),
id_skip=('noskip' not in block_string),
se_ratio=float(options['se'])if 'se' in options else None,
stride=[int(options['s'][0])])
 @staticmethod
def _encode_block_string(block):

 args = [
'r%d' % block.num_repeat,
'k%d' % block.kernel_size,
's%d%d' %(block.strides[0], block.strides[1]),
'e%s' % block.expand_ratio,
'i%d' % block.input_filters,
'o%d' % block.output_filters
]
if 0 < block.se_ratio <= 1:
args.append('se%s' % block.se_ratio)
 if block.id_skip is False:
args.append('noskip')
 return '_'.join(args)
 @staticmethod
def decode(string_list):

 assert isinstance(string_list, list)
 blocks_args = []
for block_string in string_list:
blocks_args.append(BlockDecoder._decode_block_string(block_string))
return blocks_args
@staticmethod
def encode(blocks_args):

 block_strings = []
for block in blocks_args:
block_strings.append(BlockDecoder._encode_block_string(block))
return block_strings
def efficientnet(width_coefficient=None, depth_coefficient=None, dropout_rate=0.2,
drop_connect_rate=0.2, image_size=None, num_classes=1000):

 blocks_args = [
'r1_k3_s11_e1_i32_o16_se0.25', 'r2_k3_s22_e6_i16_o24_se0.25',
'r2_k5_s22_e6_i24_o40_se0.25', 'r3_k3_s22_e6_i40_o80_se0.25',
'r3_k5_s11_e6_i80_o112_se0.25', 'r4_k5_s22_e6_i112_o192_se0.25',
'r1_k3_s11_e6_i192_o320_se0.25',
]
blocks_args = BlockDecoder.decode(blocks_args)
 global_params = GlobalParams(
 batch_norm_momentum=0.99,
batch_norm_epsilon=1e-3,
dropout_rate=dropout_rate,
drop_connect_rate=drop_connect_rate,
num_classes=num_classes,
width_coefficient=width_coefficient,
depth_coefficient=depth_coefficient,
depth_divisor=8,
min_depth=None,
image_size=image_size, 
)
 return blocks_args, global_params
def get_model_params(model_name, override_params):

 if model_name.startswith('efficientnet'):
w, d, s, p = efficientnet_params(model_name)
 blocks_args, global_params = efficientnet(
 width_coefficient=w, depth_coefficient=d, dropout_rate=p, image_size=s)
 else:
raise NotImplementedError('model name is not pre-defined: %s' % model_name)
 if override_params:
global_params = global_params._replace(**override_params)
 return blocks_args, global_params
url_map = {
'efficientnet-b0': 'http://storage.googleapis.com/public-models/efficientnet-b0-08094119.pth',
'efficientnet-b1': 'http://storage.googleapis.com/public-models/efficientnet-b1-dbc7070a.pth',
'efficientnet-b2': 'http://storage.googleapis.com/public-models/efficientnet-b2-27687264.pth',
'efficientnet-b3': 'http://storage.googleapis.com/public-models/efficientnet-b3-c8376fa2.pth',
'efficientnet-b4': 'http://storage.googleapis.com/public-models/efficientnet-b4-e116e8b3.pth',
'efficientnet-b5': 'http://storage.googleapis.com/public-models/efficientnet-b5-586e6cc6.pth',
}
def load_pretrained_weights(model, model_name, load_fc=True):

 state_dict = model_zoo.load_url(url_map[model_name])
 if load_fc:
model.load_state_dict(state_dict)
 else:
state_dict.pop('_fc.weight')
 state_dict.pop('_fc.bias')
 res = model.load_state_dict(state_dict, strict=False)
 assert str(res.missing_keys)== str(['_fc.weight', '_fc.bias']), 'issue loading pretrained weights'
print('Loaded pretrained weights for {}'.format(model_name))
class MBConvBlock(nn.Module):

 def __init__(self, block_args, global_params):
super().__init__()
self._block_args = block_args
self._bn_mom = 1 - global_params.batch_norm_momentum
self._bn_eps = global_params.batch_norm_epsilon
self.has_se =(self._block_args.se_ratio is not None)and(0 < self._block_args.se_ratio <= 1)
 self.id_skip = block_args.id_skip
Conv2d = get_same_padding_conv2d(image_size=global_params.image_size)
 inp = self._block_args.input_filters
oup = self._block_args.input_filters * self._block_args.expand_ratio
if self._block_args.expand_ratio != 1:
self._expand_conv = Conv2d(in_channels=inp, out_channels=oup, kernel_size=1, bias=False)
 self._bn0 = nn.BatchNorm2d(num_features=oup, momentum=self._bn_mom, eps=self._bn_eps)
 k = self._block_args.kernel_size
s = self._block_args.stride
self._depthwise_conv = Conv2d(
 in_channels=oup, out_channels=oup, groups=oup,
kernel_size=k, stride=s, bias=False)
 self._bn1 = nn.BatchNorm2d(num_features=oup, momentum=self._bn_mom, eps=self._bn_eps)
 if self.has_se:
num_squeezed_channels = max(1, int(self._block_args.input_filters * self._block_args.se_ratio))
self._se_reduce = Conv2d(in_channels=oup, out_channels=num_squeezed_channels, kernel_size=1)
 self._se_expand = Conv2d(in_channels=num_squeezed_channels, out_channels=oup, kernel_size=1)
 final_oup = self._block_args.output_filters
self._project_conv = Conv2d(in_channels=oup, out_channels=final_oup, kernel_size=1, bias=False)
 self._bn2 = nn.BatchNorm2d(num_features=final_oup, momentum=self._bn_mom, eps=self._bn_eps)
 def forward(self, inputs, drop_connect_rate=None):

 x = inputs
if self._block_args.expand_ratio != 1:
x = relu_fn(self._bn0(self._expand_conv(inputs)))
 x = relu_fn(self._bn1(self._depthwise_conv(x)))
 if self.has_se:
x_squeezed = F.adaptive_avg_pool2d(x, 1)
 x_squeezed = self._se_expand(relu_fn(self._se_reduce(x_squeezed)))
 x = torch.sigmoid(x_squeezed)* x
x = self._bn2(self._project_conv(x))
input_filters, output_filters = self._block_args.input_filters, self._block_args.output_filters
if self.id_skip and self._block_args.stride == 1 and input_filters == output_filters:
if drop_connect_rate:
x = drop_connect(x, p=drop_connect_rate, training=self.training)
 x = x + inputs
return x
class EfficientNet(nn.Module):

 def __init__(self, blocks_args=None, global_params=None):
super().__init__()
assert isinstance(blocks_args, list), 'blocks_args should be a list'
assert len(blocks_args)> 0, 'block args must be greater than 0'
self._global_params = global_params
self._blocks_args = blocks_args
Conv2d = get_same_padding_conv2d(image_size=global_params.image_size)
 bn_mom = 1 - self._global_params.batch_norm_momentum
bn_eps = self._global_params.batch_norm_epsilon
in_channels = 3
out_channels = round_filters(32, self._global_params)
 self._conv_stem = Conv2d(in_channels, out_channels, kernel_size=3, stride=2, bias=False)
 self._bn0 = nn.BatchNorm2d(num_features=out_channels, momentum=bn_mom, eps=bn_eps)
 self._blocks = nn.ModuleList([])
 for block_args in self._blocks_args:
block_args = block_args._replace(
 input_filters=round_filters(block_args.input_filters, self._global_params),
output_filters=round_filters(block_args.output_filters, self._global_params),
num_repeat=round_repeats(block_args.num_repeat, self._global_params)
)
 self._blocks.append(MBConvBlock(block_args, self._global_params))
if block_args.num_repeat > 1:
block_args = block_args._replace(input_filters=block_args.output_filters, stride=1)
 for _ in range(block_args.num_repeat - 1):
self._blocks.append(MBConvBlock(block_args, self._global_params))
in_channels = block_args.output_filters
out_channels = round_filters(1280, self._global_params)
 self._conv_head = Conv2d(in_channels, out_channels, kernel_size=1, bias=False)
 self._bn1 = nn.BatchNorm2d(num_features=out_channels, momentum=bn_mom, eps=bn_eps)
 self._dropout = self._global_params.dropout_rate
self._fc = nn.Linear(out_channels, self._global_params.num_classes)
 def extract_features(self, inputs):

 x = relu_fn(self._bn0(self._conv_stem(inputs)))
 for idx, block in enumerate(self._blocks):
drop_connect_rate = self._global_params.drop_connect_rate
if drop_connect_rate:
drop_connect_rate *= float(idx)/ len(self._blocks)
 x = block(x, drop_connect_rate=drop_connect_rate)
 x = relu_fn(self._bn1(self._conv_head(x)))
 return x
def forward(self, inputs):

 x = self.extract_features(inputs)
 x = F.adaptive_avg_pool2d(x, 1 ).squeeze(-1 ).squeeze(-1)
 if self._dropout:
x = F.dropout(x, p=self._dropout, training=self.training)
 x = self._fc(x)
 return x
@classmethod
def from_name(cls, model_name, override_params=None):
cls._check_model_name_is_valid(model_name)
 blocks_args, global_params = get_model_params(model_name, override_params)
 return EfficientNet(blocks_args, global_params)
 @classmethod
def from_pretrained(cls, model_name, num_classes=1000):
model = EfficientNet.from_name(model_name, override_params={'num_classes': num_classes})
 return model
@classmethod
def get_image_size(cls, model_name):
cls._check_model_name_is_valid(model_name)
 _, _, res, _ = efficientnet_params(model_name)
 return res
@classmethod
def _check_model_name_is_valid(cls, model_name, also_need_pretrained_weights=False):

 num_models = 4 if also_need_pretrained_weights else 8
valid_models = ['efficientnet_b'+str(i)for i in range(num_models)]
if model_name.replace('-','_')not in valid_models:
raise ValueError('model_name should be one of: ' + ', '.join(valid_models))
warnings.filterwarnings('ignore')
 sub0 = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/submission.csv')
 train = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/train.csv', parse_dates=['Date'])
 test = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/test.csv', parse_dates=['Date'])
 train.shape, test.shape<merge>
bs = 64
sz = 224
tfms = get_transforms(do_flip=True,flip_vert=True)
 data4 =(ImageList.from_df(df=df,path='./',cols='path')
.split_by_rand_pct(0.2)
.label_from_df(cols='diagnosis',label_cls=FloatList)
.transform(tfms,size=sz,resize_method=ResizeMethod.SQUISH,padding_mode='zeros')
.databunch(bs=bs,num_workers=4)
.normalize(imagenet_stats)
)
 learn4 = Learner(data4,
EfficientNet.from_pretrained('efficientnet-b5', num_classes=1),
metrics = [qk],
model_dir="models" ).to_fp16()
learn4.data.add_test(ImageList.from_df(test_df,
'.. /input/aptos2019-blindness-detection',
folder='test_images',
suffix='.png'))
learn4.load('abcdef');
preds4,y = learn4.get_preds(DatasetType.Test)
 del learn4, data4
know = test[test.Date <= train.Date.max() ]
not_know = test[test.Date > train.Date.max() ]
know = know.merge(train, on=['Date','Country_Region','Province_State'], how='left')
 know.head()<feature_engineering>
gc.collect() ;
train['days'] =(train['Date'] - train.Date.min() ).dt.days
train['location'] = train['Country_Region'] + train['Province_State'].fillna('')
 not_know['days'] =(not_know['Date'] - train.Date.min() ).dt.days
not_know['location'] = not_know['Country_Region'] + not_know['Province_State'].fillna('' )<feature_engineering>
class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
ll = metrics.cohen_kappa_score(y, X_p, weights='quadratic')
 return -ll
def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X=X, y=y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 print(-loss_partial(self.coef_['x']))
def predict(self, X, coef):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
return X_p
def coefficients(self):
return self.coef_['x']
def get_param(loc):
_ = train[train.location == loc]
_['diff'] = _.ConfirmedCases.diff()
_['pct'] = _.ConfirmedCases.pct_change()
initial_speed = _.loc[_.ConfirmedCases.diff().argmax() ,'pct']
initial_mid = _.loc[_.ConfirmedCases.diff().argmax() , 'days']
initial_max = _.ConfirmedCases.max() * 2
return initial_speed, initial_mid, initial_max
get_param('Italy' )<train_on_grid>
pred =(preds1+preds2+preds3+preds4+preds5+preds6)/6
<merge><EOS> 
def run_subm(preds, coefficients=[0.5, 1.5, 2.5, 3.5]):
opt = OptimizedRounder()
tst_pred = opt.predict(preds, coefficients)
 test_df.diagnosis = tst_pred.astype(int)
 test_df.to_csv('submission.csv',index=False)
 print('done' )