Kaggle competition: COVID19 Global Forecasting (Week 4)
<SOS> metric: MCRMSLE Kaggle data source: covid19-global-forecasting-(week-4)<import_modules>
import copy
import os
from collections import OrderedDict
import numpy as np
import pandas as pd
import torch
import torch.nn as nn
import torch.nn.functional as F
from matplotlib import pyplot as plt
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import MinMaxScaler
from torch.optim import AdamW
from torch.optim.lr_scheduler import ExponentialLR, ReduceLROnPlateau
from torch.utils.data import DataLoader, Dataset
from tqdm.notebook import tqdm
import torch
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from matplotlib import style
import seaborn as sns
from sklearn.model_selection import StratifiedKFold
from joblib import load, dump
from sklearn.metrics import cohen_kappa_score
from sklearn.metrics import confusion_matrix
from fastai import *
from fastai.vision import *
from fastai.callbacks import *
from torchvision import models as md
from torch import nn
from torch.nn import functional as F
import re
import math
import collections
from functools import partial
from torch.utils import model_zoo
from sklearn import metrics
from collections import Counter
import json<categorify>
main_data_dir = ".. /input/covid19-global-forecasting-week-4"
metadata_dir = ".. /input/covid19-countrywise-metadata"
GlobalParams = collections.namedtuple('GlobalParams', [
'batch_norm_momentum', 'batch_norm_epsilon', 'dropout_rate',
'num_classes', 'width_coefficient', 'depth_coefficient',
'depth_divisor', 'min_depth', 'drop_connect_rate', 'image_size'])
 BlockArgs = collections.namedtuple('BlockArgs', [
'kernel_size', 'num_repeat', 'input_filters', 'output_filters',
'expand_ratio', 'id_skip', 'stride', 'se_ratio'])
 GlobalParams.__new__.__defaults__ =(None,)* len(GlobalParams._fields)
 BlockArgs.__new__.__defaults__ =(None,)* len(BlockArgs._fields)
 def relu_fn(x):

 return x * torch.sigmoid(x)
 def round_filters(filters, global_params):

 multiplier = global_params.width_coefficient
if not multiplier:
return filters
divisor = global_params.depth_divisor
min_depth = global_params.min_depth
filters *= multiplier
min_depth = min_depth or divisor
new_filters = max(min_depth, int(filters + divisor / 2)// divisor * divisor)
 if new_filters < 0.9 * filters:
new_filters += divisor
return int(new_filters)
 def round_repeats(repeats, global_params):

 multiplier = global_params.depth_coefficient
if not multiplier:
return repeats
return int(math.ceil(multiplier * repeats))
def drop_connect(inputs, p, training):

 if not training: return inputs
batch_size = inputs.shape[0]
keep_prob = 1 - p
random_tensor = keep_prob
random_tensor += torch.rand([batch_size, 1, 1, 1], dtype=inputs.dtype, device=inputs.device)
 binary_tensor = torch.floor(random_tensor)
 output = inputs / keep_prob * binary_tensor
return output
def get_same_padding_conv2d(image_size=None):

 if image_size is None:
return Conv2dDynamicSamePadding
else:
return partial(Conv2dStaticSamePadding, image_size=image_size)
 class Conv2dDynamicSamePadding(nn.Conv2d):

 def __init__(self, in_channels, out_channels, kernel_size, stride=1, dilation=1, groups=1, bias=True):
super().__init__(in_channels, out_channels, kernel_size, stride, 0, dilation, groups, bias)
 self.stride = self.stride if len(self.stride)== 2 else [self.stride[0]]*2
def forward(self, x):
ih, iw = x.size() [-2:]
kh, kw = self.weight.size() [-2:]
sh, sw = self.stride
oh, ow = math.ceil(ih / sh), math.ceil(iw / sw)
 pad_h = max(( oh - 1)* self.stride[0] +(kh - 1)* self.dilation[0] + 1 - ih, 0)
 pad_w = max(( ow - 1)* self.stride[1] +(kw - 1)* self.dilation[1] + 1 - iw, 0)
 if pad_h > 0 or pad_w > 0:
x = F.pad(x, [pad_w//2, pad_w - pad_w//2, pad_h//2, pad_h - pad_h//2])
 return F.conv2d(x, self.weight, self.bias, self.stride, self.padding, self.dilation, self.groups)
 class Conv2dStaticSamePadding(nn.Conv2d):

 def __init__(self, in_channels, out_channels, kernel_size, image_size=None, **kwargs):
super().__init__(in_channels, out_channels, kernel_size, **kwargs)
 self.stride = self.stride if len(self.stride)== 2 else [self.stride[0]] * 2
assert image_size is not None
ih, iw = image_size if type(image_size)== list else [image_size, image_size]
kh, kw = self.weight.size() [-2:]
sh, sw = self.stride
oh, ow = math.ceil(ih / sh), math.ceil(iw / sw)
 pad_h = max(( oh - 1)* self.stride[0] +(kh - 1)* self.dilation[0] + 1 - ih, 0)
 pad_w = max(( ow - 1)* self.stride[1] +(kw - 1)* self.dilation[1] + 1 - iw, 0)
 if pad_h > 0 or pad_w > 0:
self.static_padding = nn.ZeroPad2d(( pad_w // 2, pad_w - pad_w // 2, pad_h // 2, pad_h - pad_h // 2))
else:
self.static_padding = Identity()
def forward(self, x):
x = self.static_padding(x)
 x = F.conv2d(x, self.weight, self.bias, self.stride, self.padding, self.dilation, self.groups)
 return x
class Identity(nn.Module):
def __init__(self,):
super(Identity, self ).__init__()
def forward(self, input):
return input
def efficientnet_params(model_name):

 params_dict = {
'efficientnet-b0':(1.0, 1.0, 224, 0.2),
'efficientnet-b1':(1.0, 1.1, 240, 0.2),
'efficientnet-b2':(1.1, 1.2, 260, 0.3),
'efficientnet-b3':(1.2, 1.4, 300, 0.3),
'efficientnet-b4':(1.4, 1.8, 380, 0.4),
'efficientnet-b5':(1.6, 2.2, 456, 0.4),
'efficientnet-b6':(1.8, 2.6, 528, 0.5),
'efficientnet-b7':(2.0, 3.1, 600, 0.5),
}
return params_dict[model_name]
class BlockDecoder(object):

 @staticmethod
def _decode_block_string(block_string):

 assert isinstance(block_string, str)
 ops = block_string.split('_')
 options = {}
for op in ops:
splits = re.split(r'(\d.*)', op)
 if len(splits)>= 2:
key, value = splits[:2]
options[key] = value
assert(( 's' in options and len(options['s'])== 1)or 
(len(options['s'])== 2 and options['s'][0] == options['s'][1]))
return BlockArgs(
 kernel_size=int(options['k']),
num_repeat=int(options['r']),
input_filters=int(options['i']),
output_filters=int(options['o']),
expand_ratio=int(options['e']),
id_skip=('noskip' not in block_string),
se_ratio=float(options['se'])if 'se' in options else None,
stride=[int(options['s'][0])])
 @staticmethod
def _encode_block_string(block):

 args = [
'r%d' % block.num_repeat,
'k%d' % block.kernel_size,
's%d%d' %(block.strides[0], block.strides[1]),
'e%s' % block.expand_ratio,
'i%d' % block.input_filters,
'o%d' % block.output_filters
]
if 0 < block.se_ratio <= 1:
args.append('se%s' % block.se_ratio)
 if block.id_skip is False:
args.append('noskip')
 return '_'.join(args)
 @staticmethod
def decode(string_list):

 assert isinstance(string_list, list)
 blocks_args = []
for block_string in string_list:
blocks_args.append(BlockDecoder._decode_block_string(block_string))
return blocks_args
@staticmethod
def encode(blocks_args):

 block_strings = []
for block in blocks_args:
block_strings.append(BlockDecoder._encode_block_string(block))
return block_strings
def efficientnet(width_coefficient=None, depth_coefficient=None, dropout_rate=0.2,
drop_connect_rate=0.2, image_size=None, num_classes=1000):

 blocks_args = [
'r1_k3_s11_e1_i32_o16_se0.25', 'r2_k3_s22_e6_i16_o24_se0.25',
'r2_k5_s22_e6_i24_o40_se0.25', 'r3_k3_s22_e6_i40_o80_se0.25',
'r3_k5_s11_e6_i80_o112_se0.25', 'r4_k5_s22_e6_i112_o192_se0.25',
'r1_k3_s11_e6_i192_o320_se0.25',
]
blocks_args = BlockDecoder.decode(blocks_args)
 global_params = GlobalParams(
 batch_norm_momentum=0.99,
batch_norm_epsilon=1e-3,
dropout_rate=dropout_rate,
drop_connect_rate=drop_connect_rate,
num_classes=num_classes,
width_coefficient=width_coefficient,
depth_coefficient=depth_coefficient,
depth_divisor=8,
min_depth=None,
image_size=image_size, 
)
 return blocks_args, global_params
def get_model_params(model_name, override_params):

 if model_name.startswith('efficientnet'):
w, d, s, p = efficientnet_params(model_name)
 blocks_args, global_params = efficientnet(
 width_coefficient=w, depth_coefficient=d, dropout_rate=p, image_size=s)
 else:
raise NotImplementedError('model name is not pre-defined: %s' % model_name)
 if override_params:
global_params = global_params._replace(**override_params)
 return blocks_args, global_params
url_map = {
'efficientnet-b0': 'http://storage.googleapis.com/public-models/efficientnet-b0-08094119.pth',
'efficientnet-b1': 'http://storage.googleapis.com/public-models/efficientnet-b1-dbc7070a.pth',
'efficientnet-b2': 'http://storage.googleapis.com/public-models/efficientnet-b2-27687264.pth',
'efficientnet-b3': 'http://storage.googleapis.com/public-models/efficientnet-b3-c8376fa2.pth',
'efficientnet-b4': 'http://storage.googleapis.com/public-models/efficientnet-b4-e116e8b3.pth',
'efficientnet-b5': 'http://storage.googleapis.com/public-models/efficientnet-b5-586e6cc6.pth',
}
def load_pretrained_weights(model, model_name, load_fc=True):

 state_dict = model_zoo.load_url(url_map[model_name])
 if load_fc:
model.load_state_dict(state_dict)
 else:
state_dict.pop('_fc.weight')
 state_dict.pop('_fc.bias')
 res = model.load_state_dict(state_dict, strict=False)
 assert str(res.missing_keys)== str(['_fc.weight', '_fc.bias']), 'issue loading pretrained weights'
print('Loaded pretrained weights for {}'.format(model_name))
class MBConvBlock(nn.Module):

 def __init__(self, block_args, global_params):
super().__init__()
self._block_args = block_args
self._bn_mom = 1 - global_params.batch_norm_momentum
self._bn_eps = global_params.batch_norm_epsilon
self.has_se =(self._block_args.se_ratio is not None)and(0 < self._block_args.se_ratio <= 1)
 self.id_skip = block_args.id_skip
Conv2d = get_same_padding_conv2d(image_size=global_params.image_size)
 inp = self._block_args.input_filters
oup = self._block_args.input_filters * self._block_args.expand_ratio
if self._block_args.expand_ratio != 1:
self._expand_conv = Conv2d(in_channels=inp, out_channels=oup, kernel_size=1, bias=False)
 self._bn0 = nn.BatchNorm2d(num_features=oup, momentum=self._bn_mom, eps=self._bn_eps)
 k = self._block_args.kernel_size
s = self._block_args.stride
self._depthwise_conv = Conv2d(
 in_channels=oup, out_channels=oup, groups=oup,
kernel_size=k, stride=s, bias=False)
 self._bn1 = nn.BatchNorm2d(num_features=oup, momentum=self._bn_mom, eps=self._bn_eps)
 if self.has_se:
num_squeezed_channels = max(1, int(self._block_args.input_filters * self._block_args.se_ratio))
self._se_reduce = Conv2d(in_channels=oup, out_channels=num_squeezed_channels, kernel_size=1)
 self._se_expand = Conv2d(in_channels=num_squeezed_channels, out_channels=oup, kernel_size=1)
 final_oup = self._block_args.output_filters
self._project_conv = Conv2d(in_channels=oup, out_channels=final_oup, kernel_size=1, bias=False)
 self._bn2 = nn.BatchNorm2d(num_features=final_oup, momentum=self._bn_mom, eps=self._bn_eps)
 def forward(self, inputs, drop_connect_rate=None):

 x = inputs
if self._block_args.expand_ratio != 1:
x = relu_fn(self._bn0(self._expand_conv(inputs)))
 x = relu_fn(self._bn1(self._depthwise_conv(x)))
 if self.has_se:
x_squeezed = F.adaptive_avg_pool2d(x, 1)
 x_squeezed = self._se_expand(relu_fn(self._se_reduce(x_squeezed)))
 x = torch.sigmoid(x_squeezed)* x
x = self._bn2(self._project_conv(x))
input_filters, output_filters = self._block_args.input_filters, self._block_args.output_filters
if self.id_skip and self._block_args.stride == 1 and input_filters == output_filters:
if drop_connect_rate:
x = drop_connect(x, p=drop_connect_rate, training=self.training)
 x = x + inputs
return x
class EfficientNet(nn.Module):

 def __init__(self, blocks_args=None, global_params=None):
super().__init__()
assert isinstance(blocks_args, list), 'blocks_args should be a list'
assert len(blocks_args)> 0, 'block args must be greater than 0'
self._global_params = global_params
self._blocks_args = blocks_args
Conv2d = get_same_padding_conv2d(image_size=global_params.image_size)
 bn_mom = 1 - self._global_params.batch_norm_momentum
bn_eps = self._global_params.batch_norm_epsilon
in_channels = 3
out_channels = round_filters(32, self._global_params)
 self._conv_stem = Conv2d(in_channels, out_channels, kernel_size=3, stride=2, bias=False)
 self._bn0 = nn.BatchNorm2d(num_features=out_channels, momentum=bn_mom, eps=bn_eps)
 self._blocks = nn.ModuleList([])
 for block_args in self._blocks_args:
block_args = block_args._replace(
 input_filters=round_filters(block_args.input_filters, self._global_params),
output_filters=round_filters(block_args.output_filters, self._global_params),
num_repeat=round_repeats(block_args.num_repeat, self._global_params)
)
 self._blocks.append(MBConvBlock(block_args, self._global_params))
if block_args.num_repeat > 1:
block_args = block_args._replace(input_filters=block_args.output_filters, stride=1)
 for _ in range(block_args.num_repeat - 1):
self._blocks.append(MBConvBlock(block_args, self._global_params))
in_channels = block_args.output_filters
out_channels = round_filters(1280, self._global_params)
 self._conv_head = Conv2d(in_channels, out_channels, kernel_size=1, bias=False)
 self._bn1 = nn.BatchNorm2d(num_features=out_channels, momentum=bn_mom, eps=bn_eps)
 self._dropout = self._global_params.dropout_rate
self._fc = nn.Linear(out_channels, self._global_params.num_classes)
 def extract_features(self, inputs):

 x = relu_fn(self._bn0(self._conv_stem(inputs)))
 for idx, block in enumerate(self._blocks):
drop_connect_rate = self._global_params.drop_connect_rate
if drop_connect_rate:
drop_connect_rate *= float(idx)/ len(self._blocks)
 x = block(x, drop_connect_rate=drop_connect_rate)
 x = relu_fn(self._bn1(self._conv_head(x)))
 return x
def forward(self, inputs):

 x = self.extract_features(inputs)
 x = F.adaptive_avg_pool2d(x, 1 ).squeeze(-1 ).squeeze(-1)
 if self._dropout:
x = F.dropout(x, p=self._dropout, training=self.training)
 x = self._fc(x)
 return x
@classmethod
def from_name(cls, model_name, override_params=None):
cls._check_model_name_is_valid(model_name)
 blocks_args, global_params = get_model_params(model_name, override_params)
 return EfficientNet(blocks_args, global_params)
 @classmethod
def from_pretrained(cls, model_name, num_classes=1000):
model = EfficientNet.from_name(model_name, override_params={'num_classes': num_classes})
 return model
@classmethod
def get_image_size(cls, model_name):
cls._check_model_name_is_valid(model_name)
 _, _, res, _ = efficientnet_params(model_name)
 return res
@classmethod
def _check_model_name_is_valid(cls, model_name, also_need_pretrained_weights=False):

 num_models = 4 if also_need_pretrained_weights else 8
valid_models = ['efficientnet_b'+str(i)for i in range(num_models)]
if model_name.replace('-','_')not in valid_models:
raise ValueError('model_name should be one of: ' + ', '.join(valid_models))
md_ef = EfficientNet.from_pretrained('efficientnet-b5', num_classes=1)
 !mkdir models
!cp '.. /input/kaggle-public-copy/abcdef.pth' 'models'
def get_df() :
base_image_dir = os.path.join('.. ', 'input/aptos2019-blindness-detection/')
 train_dir = os.path.join(base_image_dir,'train_images/')
 df = pd.read_csv(os.path.join(base_image_dir, 'train.csv'))
df['path'] = df['id_code'].map(lambda x: os.path.join(train_dir,'{}.png'.format(x)))
 df = df.drop(columns=['id_code'])
 df = df.sample(frac=1 ).reset_index(drop=True)
 test_df = pd.read_csv('.. /input/aptos2019-blindness-detection/sample_submission.csv')
 return df, test_df
df, test_df = get_df()
bs = 64
sz = 224
tfms = get_transforms(do_flip=True,flip_vert=True)
 data =(ImageList.from_df(df=df,path='./',cols='path')
.split_by_rand_pct(0.2)
.label_from_df(cols='diagnosis',label_cls=FloatList)
.transform(tfms,size=sz,resize_method=ResizeMethod.SQUISH,padding_mode='zeros')
.databunch(bs=bs,num_workers=4)
.normalize(imagenet_stats)
)
 def qk(y_pred, y):
return torch.tensor(cohen_kappa_score(torch.round(y_pred), y, weights='quadratic'), device='cuda:0')
 learn = Learner(data,
md_ef,
metrics = [qk],
model_dir="models" ).to_fp16()
learn.data.add_test(ImageList.from_df(test_df,
'.. /input/aptos2019-blindness-detection',
folder='test_images',
suffix='.png'))
learn.load('abcdef');
class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
ll = metrics.cohen_kappa_score(y, X_p, weights='quadratic')
 return -ll
def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X=X, y=y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 print(-loss_partial(self.coef_['x']))
def predict(self, X, coef):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
return X_p
def coefficients(self):
return self.coef_['x']
coefficients=[0.5, 1.5, 2.5, 3.5]
opt = OptimizedRounder()
preds,y = learn.get_preds(DatasetType.Test)
 preds4 = np.copy(preds)
 tst_pred = opt.predict(preds, coefficients)
 test_df.diagnosis = tst_pred.astype(int)
 submission4 = test_df.copy()
print('done' )<load_from_csv>
data_train = pd.read_csv(os.path.join(main_data_dir, "train.csv"))
data_test = pd.read_csv(os.path.join(main_data_dir, "test.csv"))
sample_sub = pd.read_csv(os.path.join(main_data_dir, "submission.csv"))
metadata = pd.read_csv(os.path.join(metadata_dir, "covid19_countrywise_metadata.csv"))
<define_search_space>
def generate_loc(x):
if isinstance(x["Province_State"], float):
return x["Country_Region"]
else:
return "_".join([x["Country_Region"], x["Province_State"]] )
score = [0.777, 0.758, 0.783]
weight = [0.35, 0.2, 0.45]
predsData = weight[0]*preds1 + weight[1]*preds2 + weight[2]*preds4<save_to_csv>
def generate_loc_date(x):
return "_".join([x["location"], x["Date"]] )
coef = [0.5, 1.5, 2.5, 3.5]
for i, pred in enumerate(predsData):
if pred < coef[0]:
predsData[i] = 0
elif pred >= coef[0] and pred < coef[1]:
predsData[i] = 1
elif pred >= coef[1] and pred < coef[2]:
predsData[i] = 2
elif pred >= coef[2] and pred < coef[3]:
predsData[i] = 3
else:
predsData[i] = 4
submission = pd.read_csv(".. /input/aptos2019-blindness-detection/sample_submission.csv")
 submission.diagnosis = predsData.astype(int)
 submission.to_csv('submission.csv', index=False )<import_modules>
data_train["location"] = data_train.apply(generate_loc, axis=1)
 data_test["location"] = data_test.apply(generate_loc, axis=1 )
import os
from keras.applications.xception import Xception
from keras.models import Model
from keras.layers import Input, Activation, Dropout, Flatten, Dense, GlobalAveragePooling2D
from keras import optimizers
from keras.callbacks import ModelCheckpoint
from keras.utils import np_utils
from keras.models import load_model
import cv2
from keras import backend as K
import matplotlib.pyplot as plt
from matplotlib.colors import rgb_to_hsv, hsv_to_rgb
import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split
import gc<load_from_csv>
data_train["log_cfm"] = data_train["ConfirmedCases"].map(np.log1p)
 data_train["log_ftl"] = data_train["Fatalities"].map(np.log1p )
img = np.loadtxt(".. /input/aptos2019-blindness-detection/train.csv",
delimiter=",",
skiprows=1,
usecols=(0),
dtype = "str" 
)
 img<load_from_csv>
data_train["loc_date"] = data_train.apply(generate_loc_date, axis=1)
 data_test["loc_date"] = data_test.apply(generate_loc_date, axis=1 )
label = np.loadtxt(".. /input/aptos2019-blindness-detection/train.csv",
delimiter=",",
skiprows=1,
usecols=(1),
dtype = "int" 
)
 label<split>
data_train_by_loc = data_train.groupby("location" )
img_label_trains = []
img_label_validations = []
for i in range(4):
data_train, data_test, labels_train, labels_test = train_test_split(img, label, train_size=0.85,random_state=i*5,stratify=label)
 img_label_train = np.stack([data_train, labels_train],axis=1)
 img_label_validation = np.stack([data_test, labels_test],axis=1)
 img_label_trains.append(img_label_train)
 img_label_validations.append(img_label_validation )<count_values>
data_train["Date"] = pd.to_datetime(data_train["Date"])
 data_test["Date"] = pd.to_datetime(data_test["Date"] )
print(np.count_nonzero(img_label_trains[0][:,1] == "0"))
print(np.count_nonzero(img_label_trains[0][:,1] == "1"))
print(np.count_nonzero(img_label_trains[0][:,1] == "2"))
print(np.count_nonzero(img_label_trains[0][:,1] == "3"))
print(np.count_nonzero(img_label_trains[0][:,1] == "4"))
print(np.count_nonzero(img_label_validations[0][:,1] == "0"))
print(np.count_nonzero(img_label_validations[0][:,1] == "1"))
print(np.count_nonzero(img_label_validations[0][:,1] == "2"))
print(np.count_nonzero(img_label_validations[0][:,1] == "3"))
print(np.count_nonzero(img_label_validations[0][:,1] == "4"))<define_variables>
data_train["rel_date"] = data_train.apply(lambda x:(x["Date"] - data_train["Date"].min()), axis=1 ).dt.days
data_test["rel_date"] = data_test.apply(lambda x:(x["Date"] - data_train["Date"].min()), axis=1 ).dt.days
def data_generator(image_lines, batch_size, abs_path, img_width, img_height, data_aug):

 n = len(image_lines)
 i = 0
while True:
image_data = []
label_data = []
for b in range(batch_size):
if i==0:
np.random.shuffle(image_lines)
 image, label = get_random_data(image_lines[i], abs_path, img_width, img_height, data_aug)
 image_data.append(image)
 label_data.append(label)
 i =(i+1)% n
image_data = np.array(image_data)
 label_data = np.array(label_data)
 yield image_data, label_data
def data_generator_wrapper(image_lines, batch_size, abs_path, img_width, img_height, data_aug):
n = len(image_lines)
 if n==0 or batch_size<=0: return None
return data_generator(image_lines, batch_size, abs_path, img_width, img_height, data_aug )<define_variables>
data_train["rel_date_pct"] = data_train.rel_date / data_test.rel_date.max()
data_test["rel_date_pct"] = data_test.rel_date / data_test.rel_date.max()
img_width, img_height = 449, 449
num_train = len(img_label_trains[0])
 num_val = len(img_label_validations[0])
 batch_size = 4
print(num_train, num_val)
 abs_path = ".. /input/aptos2019-blindness-detection/train_images/"<choose_model_class>
def get_since_day_zero(row, day_zero):
cur_date = row["rel_date"]
cur_loc = row["location"]
cur_day_zero = day_zero[cur_loc]
return max(cur_date - cur_day_zero, -1 )
models = []
for i in range(4):
input_tensor = Input(shape=(img_height, img_width, 3))
xception_model = Xception(include_top=False, weights=None, input_tensor=input_tensor)
 xception_model.load_weights(".. /input/keras-pretrained-models/xception_weights_tf_dim_ordering_tf_kernels_notop.h5")
 x = xception_model.output
x = GlobalAveragePooling2D()(x)
 x = Dense(1024, activation='relu' )(x)
 x = Dropout(0.3 )(x)
 outputs = Dense(5, activation='softmax' )(x)
 model = Model(inputs=xception_model.input, outputs=outputs)
 model.compile(optimizer=optimizers.SGD(lr=0.001,momentum=0.9),
loss='categorical_crossentropy',
metrics=['accuracy'])
 model.summary()
models.append(model )<train_model>
day_zero_location = get_day_zero(data_train)
 data_train["since_day_zero"] = data_train.apply(get_since_day_zero, axis=1, day_zero=day_zero_location)
 data_test["since_day_zero"] = data_test.apply(get_since_day_zero, axis=1, day_zero=day_zero_location )
models[0].fit_generator(data_generator_wrapper(img_label_trains[0], batch_size, abs_path, img_width, img_height, True),
steps_per_epoch=max(1, num_train//batch_size),
validation_data=data_generator_wrapper(img_label_validations[0], batch_size, abs_path, img_width, img_height, True),
validation_steps=max(1, num_val//batch_size),
epochs=5,
initial_epoch=0,
class_weight=[1,4.8,1.8,9.1,6.3] )<train_model>
data_train["since_day_zero"] = data_train["since_day_zero"].map(lambda x: -0.1 if x < 0 else x / data_test.since_day_zero.max())
 data_test["since_day_zero"] = data_test["since_day_zero"].map(lambda x: -0.1 if x < 0 else x / data_test.since_day_zero.max() )
models[0].fit_generator(data_generator_wrapper(img_label_trains[0], batch_size, abs_path, img_width, img_height, True),
steps_per_epoch=max(1, num_train//batch_size),
validation_data=data_generator_wrapper(img_label_validations[0], batch_size, abs_path, img_width, img_height, True),
validation_steps=max(1, num_val//batch_size),
epochs=5,
initial_epoch=0,
class_weight=[1,4.8,1.8,9.1,6.3] )<train_model>
lockdown_cols = ["quarantine", "close_school", "close_public_place", "limit_gathering", "stay_home"]
metadata[lockdown_cols] = metadata[lockdown_cols].fillna("2099-12-31")
 for col in lockdown_cols:
metadata[col] = pd.to_datetime(metadata[col], format="%Y-%m-%d" )
models[0].fit_generator(data_generator_wrapper(img_label_trains[0], batch_size, abs_path, img_width, img_height, True),
steps_per_epoch=max(1, num_train//batch_size),
validation_data=data_generator_wrapper(img_label_validations[0], batch_size, abs_path, img_width, img_height, True),
validation_steps=max(1, num_val//batch_size),
epochs=5,
initial_epoch=0,
class_weight=[1,4.8,1.8,9.1,6.3] )<train_model>
def add_lockdown_data(df, metadata):
lockdown_cols = ["quarantine", "close_school", "close_public_place", "limit_gathering", "stay_home"]
lockdown_df = pd.DataFrame(np.zeros(( len(df), len(lockdown_cols))), columns=lockdown_cols)
 for i in range(len(df)) :
cur_date = df.Date[i]
cur_loc = df.location[i]
idx = metadata.loc[metadata.location == cur_loc, :].index.values[0]
lockdown_df.loc[i, :] =(metadata.loc[idx, lockdown_cols] <= cur_date ).astype(int)
 return pd.concat([df, lockdown_df], axis=1 )
models[0].fit_generator(data_generator_wrapper(img_label_trains[0], batch_size, abs_path, img_width, img_height, True),
steps_per_epoch=max(1, num_train//batch_size),
validation_data=data_generator_wrapper(img_label_validations[0], batch_size, abs_path, img_width, img_height, True),
validation_steps=max(1, num_val//batch_size),
epochs=5,
initial_epoch=0,
class_weight=[1,4.8,1.8,9.1,6.3] )<train_model>
data_train = add_lockdown_data(data_train, metadata)
 data_test = add_lockdown_data(data_test, metadata )
models[1].fit_generator(data_generator_wrapper(img_label_trains[1], batch_size, abs_path, img_width, img_height, True),
steps_per_epoch=max(1, num_train//batch_size),
validation_data=data_generator_wrapper(img_label_validations[1], batch_size, abs_path, img_width, img_height, True),
validation_steps=max(1, num_val//batch_size),
epochs=5,
initial_epoch=0,
class_weight=[1,4.8,1.8,9.1,6.3] )<train_model>
metadata.drop(lockdown_cols, axis=1, inplace=True )
models[1].fit_generator(data_generator_wrapper(img_label_trains[1], batch_size, abs_path, img_width, img_height, True),
steps_per_epoch=max(1, num_train//batch_size),
validation_data=data_generator_wrapper(img_label_validations[1], batch_size, abs_path, img_width, img_height, True),
validation_steps=max(1, num_val//batch_size),
epochs=5,
initial_epoch=0,
class_weight=[1,4.8,1.8,9.1,6.3] )<train_model>
def min_max_normalize(data):
scaler = MinMaxScaler()
return scaler.fit_transform(data )
models[1].fit_generator(data_generator_wrapper(img_label_trains[1], batch_size, abs_path, img_width, img_height, True),
steps_per_epoch=max(1, num_train//batch_size),
validation_data=data_generator_wrapper(img_label_validations[1], batch_size, abs_path, img_width, img_height, True),
validation_steps=max(1, num_val//batch_size),
epochs=5,
initial_epoch=0,
class_weight=[1,4.8,1.8,9.1,6.3] )<train_model>
num_cols = metadata.columns[3:]
metadata[num_cols] = min_max_normalize(metadata[num_cols] )
models[1].fit_generator(data_generator_wrapper(img_label_trains[1], batch_size, abs_path, img_width, img_height, True),
steps_per_epoch=max(1, num_train//batch_size),
validation_data=data_generator_wrapper(img_label_validations[1], batch_size, abs_path, img_width, img_height, True),
validation_steps=max(1, num_val//batch_size),
epochs=5,
initial_epoch=0,
class_weight=[1,4.8,1.8,9.1,6.3] )<train_model>
metadata.drop(["region", "country"], axis=1, inplace=True )
models[2].fit_generator(data_generator_wrapper(img_label_trains[2], batch_size, abs_path, img_width, img_height, True),
steps_per_epoch=max(1, num_train//batch_size),
validation_data=data_generator_wrapper(img_label_validations[2], batch_size, abs_path, img_width, img_height, True),
validation_steps=max(1, num_val//batch_size),
epochs=5,
initial_epoch=0,
class_weight=[1,4.8,1.8,9.1,6.3] )<train_model>
metadata.set_index("location", inplace=True )
models[2].fit_generator(data_generator_wrapper(img_label_trains[2], batch_size, abs_path, img_width, img_height, True),
steps_per_epoch=max(1, num_train//batch_size),
validation_data=data_generator_wrapper(img_label_validations[2], batch_size, abs_path, img_width, img_height, True),
validation_steps=max(1, num_val//batch_size),
epochs=5,
initial_epoch=0,
class_weight=[1,4.8,1.8,9.1,6.3] )<train_model>
data_test["log_cfm"] = 0
data_test["log_ftl"] = 0
models[2].fit_generator(data_generator_wrapper(img_label_trains[2], batch_size, abs_path, img_width, img_height, True),
steps_per_epoch=max(1, num_train//batch_size),
validation_data=data_generator_wrapper(img_label_validations[2], batch_size, abs_path, img_width, img_height, True),
validation_steps=max(1, num_val//batch_size),
epochs=5,
initial_epoch=0,
class_weight=[1,4.8,1.8,9.1,6.3] )<train_model>
data_test_known = data_test.loc[data_test.rel_date <= data_train.rel_date.max() , :].sort_values(["rel_date", "location"] ).reset_index(drop=True)
 data_test_unknown = data_test.loc[data_test.rel_date > data_train.rel_date.max() , :].sort_values(["rel_date", "location"] ).reset_index(drop=True)
 data_train.sort_values(["rel_date", "location"], inplace=True )
models[2].fit_generator(data_generator_wrapper(img_label_trains[2], batch_size, abs_path, img_width, img_height, True),
steps_per_epoch=max(1, num_train//batch_size),
validation_data=data_generator_wrapper(img_label_validations[2], batch_size, abs_path, img_width, img_height, True),
validation_steps=max(1, num_val//batch_size),
epochs=5,
initial_epoch=0,
class_weight=[1,4.8,1.8,9.1,6.3] )<train_model>
data_test_known[["log_cfm", "log_ftl"]] = data_train.loc[data_train.rel_date >= data_test.rel_date.min() , ["log_cfm", "log_ftl"]].values
models[3].fit_generator(data_generator_wrapper(img_label_trains[2], batch_size, abs_path, img_width, img_height, True),
steps_per_epoch=max(1, num_train//batch_size),
validation_data=data_generator_wrapper(img_label_validations[2], batch_size, abs_path, img_width, img_height, True),
validation_steps=max(1, num_val//batch_size),
epochs=5,
initial_epoch=0,
class_weight=[1,4.8,1.8,9.1,6.3] )<train_model>
data_test_input = pd.concat([data_test_known, data_test_unknown], axis=0)
 input_len = 30
data_test_input = pd.concat([data_train.loc[(data_train.rel_date >= data_test_unknown.rel_date.min() - input_len)\
&(data_train.rel_date < data_test_known.rel_date.min()), data_test_input.columns],
data_test_input], axis=0 ).sort_values(["rel_date", "location"] ).reset_index(drop=True )
models[3].fit_generator(data_generator_wrapper(img_label_trains[2], batch_size, abs_path, img_width, img_height, True),
steps_per_epoch=max(1, num_train//batch_size),
validation_data=data_generator_wrapper(img_label_validations[2], batch_size, abs_path, img_width, img_height, True),
validation_steps=max(1, num_val//batch_size),
epochs=5,
initial_epoch=0,
class_weight=[1,4.8,1.8,9.1,6.3] )<train_model>
class TrainDataset(Dataset):
def __init__(self, df, metadata, x_cols, y_cols, input_len=30):
self.df = df
self.metadata = metadata
self.x_cols = x_cols
self.y_cols = y_cols
self.input_len = input_len
self.loc_list = list(self.df.location.unique())
 self.num_pos_train_period = self.df.rel_date.max() - self.input_len + 1
def __len__(self):
return len(self.loc_list)* self.num_pos_train_period
def __getitem__(self, idx):
cur_loc = self.loc_list[idx // self.num_pos_train_period]
input_beg = idx % self.num_pos_train_period
input_end = input_beg + self.input_len
in_input_period =(self.df.rel_date >= input_beg)&(self.df.rel_date < input_end)
 is_cur_loc = self.df.location == cur_loc
inputs = self.df.loc[(is_cur_loc)&(in_input_period), self.x_cols].values
meta_inputs = self.metadata.loc[cur_loc, :].values
targets = self.df.loc[(is_cur_loc)&(self.df.rel_date == input_end), self.y_cols].values.reshape(( -1))
return torch.tensor(inputs, dtype=torch.float32),\
torch.tensor(meta_inputs, dtype=torch.float32),\
torch.tensor(targets, dtype=torch.float32)
 @staticmethod
def get_dataloader(dataset, batch_size, shuffle=True):
return DataLoader(dataset, batch_size=batch_size, shuffle=shuffle )
models[3].fit_generator(data_generator_wrapper(img_label_trains[2], batch_size, abs_path, img_width, img_height, True),
steps_per_epoch=max(1, num_train//batch_size),
validation_data=data_generator_wrapper(img_label_validations[2], batch_size, abs_path, img_width, img_height, True),
validation_steps=max(1, num_val//batch_size),
epochs=5,
initial_epoch=0,
class_weight=[1,4.8,1.8,9.1,6.3] )<train_model>
rand_seed = 42
loc_list = data_train.location.unique()
loc_train, loc_val = train_test_split(loc_list, test_size=0.2, random_state=rand_seed)
 data_train.set_index("location", inplace=True, drop=True)
 data_val = data_train.loc[loc_val, :]
data_train = data_train.loc[loc_train, :]
data_train.reset_index(drop=False, inplace=True)
 data_val.reset_index(drop=False, inplace=True )
models[3].fit_generator(data_generator_wrapper(img_label_trains[2], batch_size, abs_path, img_width, img_height, True),
steps_per_epoch=max(1, num_train//batch_size),
validation_data=data_generator_wrapper(img_label_validations[2], batch_size, abs_path, img_width, img_height, True),
validation_steps=max(1, num_val//batch_size),
epochs=5,
initial_epoch=0,
class_weight=[1,4.8,1.8,9.1,6.3] )<load_from_csv>
x_cols = ["log_cfm", "log_ftl", "rel_date_pct", "since_day_zero"] + lockdown_cols
y_cols = ["log_cfm", "log_ftl"]
batch_size = 512
d_in = len(x_cols)
 d_meta = len(metadata.columns)
 ds_train = TrainDataset(data_train, metadata, x_cols, y_cols, input_len)
 dl_train = TrainDataset.get_dataloader(ds_train, batch_size)
 ds_val = TrainDataset(data_val, metadata, x_cols, y_cols, input_len)
 dl_val = TrainDataset.get_dataloader(ds_val, batch_size, shuffle=False )
img_test = np.loadtxt(".. /input/aptos2019-blindness-detection/test.csv",
delimiter=",",
skiprows=1,
dtype = "str"
)<predict_on_test>
class TestDataset(Dataset):
def __init__(self, df, metadata, x_cols, test_period_len, input_len=30):
self.df = df
self.metadata = metadata
self.x_cols = x_cols
self.test_period_len = test_period_len
self.input_len = input_len
self.loc_list = list(self.df.location.unique())
 self.test_beg = self.df.rel_date.max() - test_period_len + 1
def __len__(self):
return len(self.df)- self.input_len * len(self.loc_list)
 def __getitem__(self, idx):
df_idx = idx + self.input_len * len(self.loc_list)
 cur_loc = self.df.location[df_idx]
cur_date = self.df.rel_date[df_idx]
input_beg = cur_date - self.input_len
input_end = cur_date
in_input_period =(self.df.rel_date >= input_beg)&(self.df.rel_date < input_end)
 inputs = self.df.loc[(in_input_period)&(self.df.location == cur_loc), self.x_cols].values
inputs = np.expand_dims(inputs, axis=0)
 meta_inputs = self.metadata.loc[cur_loc, :].values
meta_inputs = np.expand_dims(meta_inputs, axis=0)
 return torch.tensor(inputs, dtype=torch.float32),\
torch.tensor(meta_inputs, dtype=torch.float32)
 @staticmethod
def get_dataloader(dataset, shuffle=False):
return DataLoader(dataset, batch_size=None, shuffle=shuffle )
test_abs_path = ".. /input/aptos2019-blindness-detection/test_images/"
data = []
for i in range(len(img_test)) :
image_file = test_abs_path + img_test[i] + ".png"
seed_image = cv2.imread(image_file)
 seed_image = cv2.cvtColor(seed_image, cv2.COLOR_BGR2RGB)
 seed_image = cv2.resize(seed_image, dsize=(img_width, img_height))
seed_image = np.expand_dims(seed_image, axis=0)
 seed_image = seed_image / 255
predict1 = models[0].predict(seed_image)
 predict2 = models[1].predict(seed_image)
 predict3 = models[2].predict(seed_image)
 predict4 = models[3].predict(seed_image)
 predict_mean =(predict1+predict2+predict3+predict4)/4
x = np.array([img_test[i], np.argmax(predict_mean)])
 data.append(x)
 data = np.array(data )<create_dataframe>
test_period_len = data_test.rel_date.max() - data_test.rel_date.min() + 1
ds_test = TestDataset(data_test_input, metadata, x_cols, test_period_len)
 dl_test = TestDataset.get_dataloader(ds_test, shuffle=False )
columns = ['id_code', 'diagnosis']
name = 'sample'
d = pd.DataFrame(data=data, columns=columns, dtype='str' )<save_to_csv>
for i, sample in enumerate(dl_test):
for x in sample:
print(x.size())
 if i == 10:
break
d.to_csv("submission.csv",index=False )<load_from_csv>
class CovidTransformer(nn.Module):
def __init__(self, d_in, d_out, d_meta, d_model=256, d_fwd=512, n_head=4,
num_layers=6, dropout=0.1):
super().__init__()
self.linear = nn.Linear(d_in, d_model)
 encoder_layer = nn.TransformerEncoderLayer(d_model, n_head, d_fwd, dropout)
 self.transformer = nn.TransformerEncoder(encoder_layer, num_layers)
 self.meta_linear = nn.Linear(d_meta, d_model)
 self.regressor = nn.Linear(2 * d_model, d_out)
 def forward(self, inputs, meta_inputs):
x = self.linear(inputs)
 features = self.transformer(x)
 features = features[-1, :, :]
meta_features = self.meta_linear(meta_inputs)
 features = torch.cat([features, meta_features], dim=-1)
 output = self.regressor(features)
 return output
df = pd.read_csv("submission.csv")
 print(df )<define_variables>
class RMSLE(nn.Module):
def forward(self, output, target):
return torch.sqrt(F.mse_loss(output, target))
PATH = ".. /input/aptos2019-blindness-detection"
!mkdir -p /tmp/.cache/torch/checkpoints/
!cp.. /input/fastai-pretrained-models/resnet101-5d3b4d8f.pth /tmp/.cache/torch/checkpoints/resnet101-5d3b4d8f.pth<load_from_csv>
model = CovidTransformer(d_in, 2, d_meta)
 if torch.cuda.is_available() :
model = model.cuda()
criterion = RMSLE()
optimizer = AdamW(model.parameters())
 epochs = 20
scheduler = ExponentialLR(optimizer, 0.707 )
SIZE=224
train_df=pd.read_csv(PATH+'/train.csv')
 test_df=pd.read_csv(PATH+'/sample_submission.csv' )<define_variables>
engine = Engine()
engine.compile(model, criterion, optimizer, scheduler )
train = ImageList.from_df(train_df, path=PATH, cols='id_code', folder="train_images", suffix='.png')
 test = ImageList.from_df(test_df, path=PATH, cols='id_code', folder="test_images", suffix='.png' )<compute_test_metric>
engine.fit(epochs, dl_train, dl_val )
def quadratic_kappa(y_hat, y):
return torch.tensor(cohen_kappa_score(torch.round(y_hat), y, weights='quadratic'),device='cuda:0' )<choose_model_class>
def make_predictions(dl_test, engine, y_cols):
progress = tqdm(total=len(dl_test))
for i, sample in enumerate(dl_test):
idx = i + dl_test.dataset.input_len * len(dl_test.dataset.loc_list)
 cur_loc = dl_test.dataset.df.location[idx]
cur_date = dl_test.dataset.df.rel_date[idx]
output = engine.predict_batch(sample ).detach().cpu().numpy()
prev_date = cur_date - 1
prev_output = dl_test.dataset.df.loc[(dl_test.dataset.df.rel_date == prev_date)
 &(dl_test.dataset.df.location == cur_loc), y_cols].values
output = np.maximum(output, prev_output)
 dl_test.dataset.df.loc[(dl_test.dataset.df.rel_date == cur_date)
 &(dl_test.dataset.df.location == cur_loc), y_cols] = output
progress.update(1)
 return dl_test.dataset.df
learn = cnn_learner(data, models.resnet101, metrics=[quadratic_kappa], pretrained=True)
<train_model>
predictions = make_predictions(dl_test, engine, y_cols )
learn.fit_one_cycle(6, slice(2.75e-03,2.75e-02))
learn.recorder.plot_losses()
learn.recorder.plot_metrics() 
<save_model>
predictions["ConfirmedCases"] = predictions["log_cfm"].map(np.expm1)
 predictions["Fatalities"] = predictions["log_ftl"].map(np.expm1 )
learn.save('stage1' )<train_model>
predictions = predictions.loc[~predictions.ForecastId.isna() , :]
lr = 2.75e-03
learn.fit_one_cycle(4,slice(1.5e-06,lr/8),wd=0.05)
<prepare_x_and_y>
predictions = predictions.sort_values(["ForecastId"] ).reset_index(drop=True )
valid_preds, valid_y = learn.TTA(ds_type=DatasetType.Valid)
 test_preds, _ = learn.TTA(ds_type=DatasetType.Test )<compute_test_metric>
submission = predictions[["ForecastId", "ConfirmedCases", "Fatalities"]]
submission["ForecastId"] = submission["ForecastId"].astype(int )
class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
ll = cohen_kappa_score(y, X_p, weights='quadratic')
 return -ll
def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X=X, y=y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 def predict(self, X, coef):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
return X_p
def coefficients(self):
return self.coef_['x']<compute_train_metric>
submission.to_csv("submission.csv", index=False )
<compute_test_metric><EOS> 
submission.to_csv("submission.csv", index=False )