Kaggle competition: Prostate cANcer graDe Assessment (PANDA) Challenge
def train(
 X_data,
y_data,
params,
feature_names,
verbose_eval=50,
kfold_random_state=23948,
predict=False,
calculate_mcc=False,
early_stopping_rounds=10,
num_folds=5,
num_iterations=10, 
):
params = copy.deepcopy(params)
 models = []
cv_scores = []
val_cv_scores = []
if calculate_mcc:
mcc_scores = []
val_mcc_scores = []
thresholds=[]
else:
mcc_scores = None
val_mcc_scores = None
thresholds = None
if predict:
yp_train = np.zeros(X_train.shape[0])
 yp_val = np.zeros(X_train.shape[0])
 yp_test = np.zeros(X_train.shape[0])
 else:
yp_train = None
yp_val = None
yp_test = None
for iter in range(num_iterations):
np.random.seed(kfold_random_state + iter)
 splits = np.zeros(X_data.shape[0], dtype=np.int)
 m = y_data == 1
splits[m] = np.random.randint(0, 5, size=m.sum())
 m = y_data == 0
splits[m] = np.random.randint(0, 5, size=m.sum())
 for fold in range(num_folds):
val_fold = fold
test_fold =(fold + 1)% num_folds
train_folds = [f for f in range(num_folds)if f not in [val_fold, test_fold]]
train_indices = np.where(np.isin(splits, train_folds)) [0]
val_indices = np.where(splits == val_fold)[0]
test_indices = np.where(splits == test_fold)[0]
trn = lgb.Dataset(
 X_data.values[train_indices],
y_data[train_indices],
feature_name=feature_names, 
)
 val = lgb.Dataset(
 X_data.values[val_indices],
y_data[val_indices],
feature_name=feature_names, 
)
 test = lgb.Dataset(
 X_data.values[test_indices],
y_data[test_indices],
feature_name=feature_names, 
)
 with warnings.catch_warnings() :
warnings.simplefilter("ignore")
 model = lgb.train(
 params,
trn,
valid_sets=(trn, test, val),
valid_names=("train", "test", "validation"),
fobj=None,
feval=None,
early_stopping_rounds=early_stopping_rounds,
verbose_eval=verbose_eval, 
)
 yp = None
if predict:
yp = model.predict(X_train.values[train_indices])
 yp_train[train_indices] += yp
yp_val_fold = model.predict(X_train.values[val_indices])
 yp_val[val_indices] += yp_val_fold
yp_test_fold = model.predict(X_train.values[test_indices])
 yp_test[test_indices] += yp_test_fold
models.append(model)
 s = model.best_score['test']
cv_scores.append(s[list(s.keys())[0]])
 s = model.best_score['validation']
val_cv_scores.append(s[list(s.keys())[0]])
 if calculate_mcc:
best_proba, _, _ = eval_mcc(
 y_train.values[train_indices].astype(np.float),
yp,
False 
)
 best_mcc_val = matthews_corrcoef(
 y_train.values[val_indices].astype(np.float),
yp_val_fold.astype(np.float)> best_proba 
)
 best_mcc = matthews_corrcoef(
 y_train.values[test_indices].astype(np.float),
yp_test_fold.astype(np.float)> best_proba 
)
 val_mcc_scores.append(best_mcc_val)
 mcc_scores.append(best_mcc)
 thresholds.append(best_proba)
 if yp_train is not None:
yp_train /=(( num_folds - 2)* num_iterations)
 yp_val /= num_iterations
yp_test /= num_iterations
cv_scores = np.array(cv_scores)
 val_cv_scores = np.array(val_cv_scores)
 if calculate_mcc:
mcc_scores = np.array(mcc_scores)
 val_mcc_scores = np.array(val_mcc_scores)
 thresholds = np.array(thresholds)
 return(
 models,
cv_scores, val_cv_scores,
mcc_scores, val_mcc_scores,
yp_train, yp_val, yp_test, thresholds
)<init_hyperparams>
!echo "class EfficientNet: pass" > efficientnet_pytorch.py
params = {
'objective': 'binary',
'boosting': 'gbdt',
'learning_rate': 0.01,
'num_leaves': 80,
'num_threads': 4,
'metric': 'binary_logloss',
'feature_fraction': 0.8,
'bagging_freq': 1,
'bagging_fraction': 0.8,
'seed': 23974,
'num_boost_round': 10000,
}<train_model>
try:
USE_APEX = True
except:
USE_APEX = False
%%time 
(
 models,
cv_scores, val_cv_scores,
mcc_scores, val_mcc_scores,
yp_train, yp_val, yp_test, thresholds 
)= train(
 X_train,
y_train,
params,
feature_names,
verbose_eval=0,
kfold_random_state=123948,
predict=True,
calculate_mcc=True,
early_stopping_rounds=100,
num_iterations=25 
)
 print("")
 print("CV Logloss: {:.4f} +/- {:.4f}({:.4f})".format(cv_scores.mean() , cv_scores.std() /np.sqrt(cv_scores.shape[0]), cv_scores.std()))
print("CV Val Logloss: {:.4f} +/- {:.4f}({:.4f})".format(val_cv_scores.mean() , val_cv_scores.std() /np.sqrt(val_cv_scores.shape[0]), val_cv_scores.std()))
print("CV MCC: {:.4f} +/- {:.4f}({:.4f})".format(mcc_scores.mean() , mcc_scores.std() /np.sqrt(mcc_scores.shape[0]), mcc_scores.std()))
print("CV VAL MCC: {:.4f} +/- {:.4f}({:.4f})".format(val_mcc_scores.mean() , val_mcc_scores.std() /np.sqrt(val_mcc_scores.shape[0]), val_mcc_scores.std()))
print("Threshold {:.4f} +/- {:.4f}".format(thresholds.mean() , thresholds.std() /np.sqrt(thresholds.shape[0])))
 best_proba_meas, _, _ = eval_mcc(
 y_train.values.astype(np.float),
yp_train,
False 
)
 best_mcc_meas = matthews_corrcoef(
 y_train.values.astype(np.float),
yp_val > best_proba_meas 
)
 best_mcc_meas_test = matthews_corrcoef(
 y_train.values.astype(np.float),
yp_test > best_proba_meas 
)
 plt.show()
yp_train_df = pd.DataFrame(
 yp_train,
index=X_train_full.index 
)
 yp_val_df = pd.DataFrame(
 yp_val,
index=X_train_full.index 
)
 yp_test_df = pd.DataFrame(
 yp_test,
index=X_train_full.index 
)
 train_pred = meta_train_df[['id_measurement', 'signal_id', 'target']].copy()
train_pred = pd.merge(train_pred, yp_train_df, on='id_measurement')
 train_pred.rename({0:'prediction'}, axis=1, inplace=True)
 val_pred = meta_train_df[['id_measurement', 'signal_id', 'target']].copy()
val_pred = pd.merge(val_pred, yp_val_df, on='id_measurement')
 val_pred.rename({0:'prediction'}, axis=1, inplace=True)
 test_pred = meta_train_df[['id_measurement', 'signal_id', 'target']].copy()
test_pred = pd.merge(test_pred, yp_test_df, on='id_measurement')
 test_pred.rename({0:'prediction'}, axis=1, inplace=True)
 best_proba, _, _ = eval_mcc(
 train_pred['target'].values.astype(np.float),
train_pred['prediction'].values.astype(np.float),
False 
)
 best_mcc = matthews_corrcoef(
 val_pred['target'].values.astype(np.float),
val_pred['prediction'].values.astype(np.float)> best_proba 
)
 best_mcc_test = matthews_corrcoef(
 test_pred['target'].values.astype(np.float),
test_pred['prediction'].values.astype(np.float)> best_proba 
)
 plt.show()
print("MCC(measurement): {:.3f}".format(best_mcc_meas))
print("MCC Test(measurement): {:.3f}".format(best_mcc_meas_test))
print("Best Probability Threshold(measurement): {:.3f}".format(best_proba_meas))
print("MCC: {:.3f}".format(best_mcc))
print("MCC Test: {:.3f}".format(best_mcc_test))
print("Best Probability Threshold: {:.3f}".format(best_proba))<categorify>
USE_PRETRAINED = False
GlobalParams = collections.namedtuple('GlobalParams', [
'batch_norm_momentum', 'batch_norm_epsilon', 'dropout_rate',
'num_classes', 'width_coefficient', 'depth_coefficient',
'depth_divisor', 'min_depth', 'drop_connect_rate', 'image_size'])
 BlockArgs = collections.namedtuple('BlockArgs', [
'kernel_size', 'num_repeat', 'input_filters', 'output_filters',
'expand_ratio', 'id_skip', 'stride', 'se_ratio'])
 GlobalParams.__new__.__defaults__ =(None,)* len(GlobalParams._fields)
 BlockArgs.__new__.__defaults__ =(None,)* len(BlockArgs._fields)
 class SwishImplementation(torch.autograd.Function):
@staticmethod
def forward(ctx, i):
result = i * torch.sigmoid(i)
 ctx.save_for_backward(i)
 return result
@staticmethod
def backward(ctx, grad_output):
i = ctx.saved_variables[0]
sigmoid_i = torch.sigmoid(i)
 return grad_output *(sigmoid_i *(1 + i *(1 - sigmoid_i)))
 class MemoryEfficientSwish(nn.Module):
def forward(self, x):
return SwishImplementation.apply(x)
 class Swish(nn.Module):
def forward(self, x):
return x * torch.sigmoid(x)
 def round_filters(filters, global_params):

 multiplier = global_params.width_coefficient
if not multiplier:
return filters
divisor = global_params.depth_divisor
min_depth = global_params.min_depth
filters *= multiplier
min_depth = min_depth or divisor
new_filters = max(min_depth, int(filters + divisor / 2)// divisor * divisor)
 if new_filters < 0.9 * filters:
new_filters += divisor
return int(new_filters)
 def round_repeats(repeats, global_params):

 multiplier = global_params.depth_coefficient
if not multiplier:
return repeats
return int(math.ceil(multiplier * repeats))
def drop_connect(inputs, p, training):

 if not training: return inputs
batch_size = inputs.shape[0]
keep_prob = 1 - p
random_tensor = keep_prob
random_tensor += torch.rand([batch_size, 1, 1, 1], dtype=inputs.dtype, device=inputs.device)
 binary_tensor = torch.floor(random_tensor)
 output = inputs / keep_prob * binary_tensor
return output
def get_same_padding_conv2d(image_size=None):

 if image_size is None:
return Conv2dDynamicSamePadding
else:
return partial(Conv2dStaticSamePadding, image_size=image_size)
 def get_width_and_height_from_size(x):

 if isinstance(x, int): return x, x
if isinstance(x, list)or isinstance(x, tuple): return x
else: raise TypeError()
def calculate_output_image_size(input_image_size, stride):

 if input_image_size is None: return None
image_height, image_width = get_width_and_height_from_size(input_image_size)
 stride = stride if isinstance(stride, int)else stride[0]
image_height = int(math.ceil(image_height / stride))
image_width = int(math.ceil(image_width / stride))
return [image_height, image_width]
class Conv2dDynamicSamePadding(nn.Conv2d):

 def __init__(self, in_channels, out_channels, kernel_size, stride=1, dilation=1, groups=1, bias=True):
super().__init__(in_channels, out_channels, kernel_size, stride, 0, dilation, groups, bias)
 self.stride = self.stride if len(self.stride)== 2 else [self.stride[0]] * 2
def forward(self, x):
ih, iw = x.size() [-2:]
kh, kw = self.weight.size() [-2:]
sh, sw = self.stride
oh, ow = math.ceil(ih / sh), math.ceil(iw / sw)
 pad_h = max(( oh - 1)* self.stride[0] +(kh - 1)* self.dilation[0] + 1 - ih, 0)
 pad_w = max(( ow - 1)* self.stride[1] +(kw - 1)* self.dilation[1] + 1 - iw, 0)
 if pad_h > 0 or pad_w > 0:
x = F.pad(x, [pad_w // 2, pad_w - pad_w // 2, pad_h // 2, pad_h - pad_h // 2])
 return F.conv2d(x, self.weight, self.bias, self.stride, self.padding, self.dilation, self.groups)
 class Conv2dStaticSamePadding(nn.Conv2d):

 def __init__(self, in_channels, out_channels, kernel_size, image_size=None, **kwargs):
super().__init__(in_channels, out_channels, kernel_size, **kwargs)
 self.stride = self.stride if len(self.stride)== 2 else [self.stride[0]] * 2
assert image_size is not None
ih, iw =(image_size, image_size)if isinstance(image_size, int)else image_size
kh, kw = self.weight.size() [-2:]
sh, sw = self.stride
oh, ow = math.ceil(ih / sh), math.ceil(iw / sw)
 pad_h = max(( oh - 1)* self.stride[0] +(kh - 1)* self.dilation[0] + 1 - ih, 0)
 pad_w = max(( ow - 1)* self.stride[1] +(kw - 1)* self.dilation[1] + 1 - iw, 0)
 if pad_h > 0 or pad_w > 0:
self.static_padding = nn.ZeroPad2d(( pad_w // 2, pad_w - pad_w // 2, pad_h // 2, pad_h - pad_h // 2))
else:
self.static_padding = Identity()
def forward(self, x):
x = self.static_padding(x)
 x = F.conv2d(x, self.weight, self.bias, self.stride, self.padding, self.dilation, self.groups)
 return x
class Identity(nn.Module):
def __init__(self,):
super(Identity, self ).__init__()
def forward(self, input):
return input
def efficientnet_params(model_name):

 params_dict = {
'efficientnet-b0':(1.0, 1.0, 224, 0.2),
'efficientnet-b1':(1.0, 1.1, 240, 0.2),
'efficientnet-b2':(1.1, 1.2, 260, 0.3),
'efficientnet-b3':(1.2, 1.4, 300, 0.3),
'efficientnet-b4':(1.4, 1.8, 380, 0.4),
'efficientnet-b5':(1.6, 2.2, 456, 0.4),
'efficientnet-b6':(1.8, 2.6, 528, 0.5),
'efficientnet-b7':(2.0, 3.1, 600, 0.5),
'efficientnet-b8':(2.2, 3.6, 672, 0.5),
'efficientnet-l2':(4.3, 5.3, 800, 0.5),
}
return params_dict[model_name]
class BlockDecoder(object):

 @staticmethod
def _decode_block_string(block_string):

 assert isinstance(block_string, str)
 ops = block_string.split('_')
 options = {}
for op in ops:
splits = re.split(r'(\d.*)', op)
 if len(splits)>= 2:
key, value = splits[:2]
options[key] = value
assert(( 's' in options and len(options['s'])== 1)or 
(len(options['s'])== 2 and options['s'][0] == options['s'][1]))
return BlockArgs(
 kernel_size=int(options['k']),
num_repeat=int(options['r']),
input_filters=int(options['i']),
output_filters=int(options['o']),
expand_ratio=int(options['e']),
id_skip=('noskip' not in block_string),
se_ratio=float(options['se'])if 'se' in options else None,
stride=[int(options['s'][0])])
 @staticmethod
def _encode_block_string(block):

 args = [
'r%d' % block.num_repeat,
'k%d' % block.kernel_size,
's%d%d' %(block.strides[0], block.strides[1]),
'e%s' % block.expand_ratio,
'i%d' % block.input_filters,
'o%d' % block.output_filters
]
if 0 < block.se_ratio <= 1:
args.append('se%s' % block.se_ratio)
 if block.id_skip is False:
args.append('noskip')
 return '_'.join(args)
 @staticmethod
def decode(string_list):

 assert isinstance(string_list, list)
 blocks_args = []
for block_string in string_list:
blocks_args.append(BlockDecoder._decode_block_string(block_string))
return blocks_args
@staticmethod
def encode(blocks_args):

 block_strings = []
for block in blocks_args:
block_strings.append(BlockDecoder._encode_block_string(block))
return block_strings
def efficientnet(width_coefficient=None, depth_coefficient=None, dropout_rate=0.2,
drop_connect_rate=0.2, image_size=None, num_classes=1000):

 blocks_args = [
'r1_k3_s11_e1_i32_o16_se0.25', 'r2_k3_s22_e6_i16_o24_se0.25',
'r2_k5_s22_e6_i24_o40_se0.25', 'r3_k3_s22_e6_i40_o80_se0.25',
'r3_k5_s11_e6_i80_o112_se0.25', 'r4_k5_s22_e6_i112_o192_se0.25',
'r1_k3_s11_e6_i192_o320_se0.25',
]
blocks_args = BlockDecoder.decode(blocks_args)
 global_params = GlobalParams(
 batch_norm_momentum=0.99,
batch_norm_epsilon=1e-3,
dropout_rate=dropout_rate,
drop_connect_rate=drop_connect_rate,
num_classes=num_classes,
width_coefficient=width_coefficient,
depth_coefficient=depth_coefficient,
depth_divisor=8,
min_depth=None,
image_size=image_size, 
)
 return blocks_args, global_params
def get_model_params(model_name, override_params):

 if model_name.startswith('efficientnet'):
w, d, s, p = efficientnet_params(model_name)
 blocks_args, global_params = efficientnet(
 width_coefficient=w, depth_coefficient=d, dropout_rate=p, image_size=s)
 else:
raise NotImplementedError('model name is not pre-defined: %s' % model_name)
 if override_params:
global_params = global_params._replace(**override_params)
 return blocks_args, global_params
url_map = {
'efficientnet-b0': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/efficientnet-b0-355c32eb.pth',
'efficientnet-b1': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/efficientnet-b1-f1951068.pth',
'efficientnet-b2': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/efficientnet-b2-8bb594d6.pth',
'efficientnet-b3': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/efficientnet-b3-5fb5a3c3.pth',
'efficientnet-b4': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/efficientnet-b4-6ed6700e.pth',
'efficientnet-b5': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/efficientnet-b5-b6417697.pth',
'efficientnet-b6': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/efficientnet-b6-c76e70fd.pth',
'efficientnet-b7': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/efficientnet-b7-dcc49843.pth',
}
url_map_advprop = {
'efficientnet-b0': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/adv-efficientnet-b0-b64d5a18.pth',
'efficientnet-b1': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/adv-efficientnet-b1-0f3ce85a.pth',
'efficientnet-b2': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/adv-efficientnet-b2-6e9d97e5.pth',
'efficientnet-b3': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/adv-efficientnet-b3-cdd7c0f4.pth',
'efficientnet-b4': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/adv-efficientnet-b4-44fb3a87.pth',
'efficientnet-b5': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/adv-efficientnet-b5-86493f6b.pth',
'efficientnet-b6': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/adv-efficientnet-b6-ac80338e.pth',
'efficientnet-b7': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/adv-efficientnet-b7-4652b6dd.pth',
'efficientnet-b8': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/adv-efficientnet-b8-22a8fe65.pth',
}
def load_pretrained_weights(model, model_name, load_fc=True, advprop=False):

 url_map_ = url_map_advprop if advprop else url_map
state_dict = model_zoo.load_url(url_map_[model_name])
 if load_fc:
model.load_state_dict(state_dict)
 else:
state_dict.pop('_fc.weight')
 state_dict.pop('_fc.bias')
 res = model.load_state_dict(state_dict, strict=False)
 assert set(res.missing_keys)== set(['_fc.weight', '_fc.bias']), 'issue loading pretrained weights'
print('Loaded pretrained weights for {}'.format(model_name))
class MBConvBlock(nn.Module):

 def __init__(self, block_args, global_params, image_size=None):
super().__init__()
self._block_args = block_args
self._bn_mom = 1 - global_params.batch_norm_momentum
self._bn_eps = global_params.batch_norm_epsilon
self.has_se =(self._block_args.se_ratio is not None)and(0 < self._block_args.se_ratio <= 1)
 self.id_skip = block_args.id_skip
inp = self._block_args.input_filters
oup = self._block_args.input_filters * self._block_args.expand_ratio
if self._block_args.expand_ratio != 1:
Conv2d = get_same_padding_conv2d(image_size=image_size)
 self._expand_conv = Conv2d(in_channels=inp, out_channels=oup, kernel_size=1, bias=False)
 self._bn0 = nn.BatchNorm2d(num_features=oup, momentum=self._bn_mom, eps=self._bn_eps)
 k = self._block_args.kernel_size
s = self._block_args.stride
Conv2d = get_same_padding_conv2d(image_size=image_size)
 self._depthwise_conv = Conv2d(
 in_channels=oup, out_channels=oup, groups=oup,
kernel_size=k, stride=s, bias=False)
 self._bn1 = nn.BatchNorm2d(num_features=oup, momentum=self._bn_mom, eps=self._bn_eps)
 image_size = calculate_output_image_size(image_size, s)
 if self.has_se:
Conv2d = get_same_padding_conv2d(image_size=(1,1))
num_squeezed_channels = max(1, int(self._block_args.input_filters * self._block_args.se_ratio))
self._se_reduce = Conv2d(in_channels=oup, out_channels=num_squeezed_channels, kernel_size=1)
 self._se_expand = Conv2d(in_channels=num_squeezed_channels, out_channels=oup, kernel_size=1)
 final_oup = self._block_args.output_filters
Conv2d = get_same_padding_conv2d(image_size=image_size)
 self._project_conv = Conv2d(in_channels=oup, out_channels=final_oup, kernel_size=1, bias=False)
 self._bn2 = nn.BatchNorm2d(num_features=final_oup, momentum=self._bn_mom, eps=self._bn_eps)
 self._swish = MemoryEfficientSwish()
def forward(self, inputs, drop_connect_rate=None):

 x = inputs
if self._block_args.expand_ratio != 1:
x = self._swish(self._bn0(self._expand_conv(inputs)))
 x = self._swish(self._bn1(self._depthwise_conv(x)))
 if self.has_se:
x_squeezed = F.adaptive_avg_pool2d(x, 1)
 x_squeezed = self._se_expand(self._swish(self._se_reduce(x_squeezed)))
 x = torch.sigmoid(x_squeezed)* x
x = self._bn2(self._project_conv(x))
input_filters, output_filters = self._block_args.input_filters, self._block_args.output_filters
if self.id_skip and self._block_args.stride == 1 and input_filters == output_filters:
if drop_connect_rate:
x = drop_connect(x, p=drop_connect_rate, training=self.training)
 x = x + inputs
return x
def set_swish(self, memory_efficient=True):

 self._swish = MemoryEfficientSwish() if memory_efficient else Swish()
class EfficientNet(nn.Module):

 def __init__(self, blocks_args=None, global_params=None):
super().__init__()
assert isinstance(blocks_args, list), 'blocks_args should be a list'
assert len(blocks_args)> 0, 'block args must be greater than 0'
self._global_params = global_params
self._blocks_args = blocks_args
bn_mom = 1 - self._global_params.batch_norm_momentum
bn_eps = self._global_params.batch_norm_epsilon
image_size = global_params.image_size
Conv2d = get_same_padding_conv2d(image_size=global_params.image_size)
 in_channels = 3
out_channels = round_filters(32, self._global_params)
 self._conv_stem = Conv2d(in_channels, out_channels, kernel_size=3, stride=2, bias=False)
 self._bn0 = nn.BatchNorm2d(num_features=out_channels, momentum=bn_mom, eps=bn_eps)
 image_size = calculate_output_image_size(image_size, 2)
 self._blocks = nn.ModuleList([])
 for block_args in self._blocks_args:
block_args = block_args._replace(
 input_filters=round_filters(block_args.input_filters, self._global_params),
output_filters=round_filters(block_args.output_filters, self._global_params),
num_repeat=round_repeats(block_args.num_repeat, self._global_params)
)
 self._blocks.append(MBConvBlock(block_args, self._global_params, image_size=image_size))
image_size = calculate_output_image_size(image_size, block_args.stride)
 if block_args.num_repeat > 1:
block_args = block_args._replace(input_filters=block_args.output_filters, stride=1)
 for _ in range(block_args.num_repeat - 1):
self._blocks.append(MBConvBlock(block_args, self._global_params, image_size=image_size))
in_channels = block_args.output_filters
out_channels = round_filters(1280, self._global_params)
 Conv2d = get_same_padding_conv2d(image_size=image_size)
 self._conv_head = Conv2d(in_channels, out_channels, kernel_size=1, bias=False)
 self._bn1 = nn.BatchNorm2d(num_features=out_channels, momentum=bn_mom, eps=bn_eps)
 self._avg_pooling = nn.AdaptiveAvgPool2d(1)
 self._dropout = nn.Dropout(self._global_params.dropout_rate)
 self._fc = nn.Linear(out_channels, self._global_params.num_classes)
 self._swish = MemoryEfficientSwish()
def set_swish(self, memory_efficient=True):

 self._swish = MemoryEfficientSwish() if memory_efficient else Swish()
for block in self._blocks:
block.set_swish(memory_efficient)
 def extract_features(self, inputs):

 x = self._swish(self._bn0(self._conv_stem(inputs)))
 for idx, block in enumerate(self._blocks):
drop_connect_rate = self._global_params.drop_connect_rate
if drop_connect_rate:
drop_connect_rate *= float(idx)/ len(self._blocks)
 x = block(x, drop_connect_rate=drop_connect_rate)
 x = self._swish(self._bn1(self._conv_head(x)))
 return x
def forward(self, inputs):

 bs = inputs.size(0)
 x = self.extract_features(inputs)
 x = self._avg_pooling(x)
 x = x.view(bs, -1)
 x = self._dropout(x)
 x = self._fc(x)
 return x
@classmethod
def from_name(cls, model_name, override_params=None):
cls._check_model_name_is_valid(model_name)
 blocks_args, global_params = get_model_params(model_name, override_params)
 return cls(blocks_args, global_params)
 @classmethod
def from_pretrained(cls, model_name, advprop=False, num_classes=1000, in_channels=3):
model = cls.from_name(model_name, override_params={'num_classes': num_classes})
 load_pretrained_weights(model, model_name, load_fc=(num_classes == 1000), advprop=advprop)
 model._change_in_channels(in_channels)
 return model
@classmethod
def get_image_size(cls, model_name):
cls._check_model_name_is_valid(model_name)
 _, _, res, _ = efficientnet_params(model_name)
 return res
@classmethod
def _check_model_name_is_valid(cls, model_name):

 valid_models = ['efficientnet-b'+str(i)for i in range(9)]
if model_name not in valid_models:
raise ValueError('model_name should be one of: ' + ', '.join(valid_models))
def _change_in_channels(model, in_channels):
if in_channels != 3:
Conv2d = get_same_padding_conv2d(image_size = model._global_params.image_size)
 out_channels = round_filters(32, model._global_params)
 model._conv_stem = Conv2d(in_channels, out_channels, kernel_size=3, stride=2, bias=False)
 class FeatureEfficientNet(EfficientNet):
def forward(self, inputs):

 bs = inputs.size(0)
 x = self.extract_features(inputs)
 return x
class PandaDataset(D.Dataset):
def __init__(self, images, labels, insts=None, img_size=2, transform=None, bin_label=False,
root_path='', istest=False, return_index=True,
use_cache=False, mixup=False, mixup_alpha=1.0, separate_image=False, cat_insts=False):
assert len(images)== len(labels)
 self.images = images
self.labels = labels
self.insts = insts
self.img_size = img_size
self.transform = transform
self.bin_label = bin_label
self.root = root_path
self.istest = istest
self.return_index = return_index
self.use_cache = use_cache
self.cache = {}
self.mixup = mixup
self.mixup_alpha = mixup_alpha
self.separate_image = separate_image
self.cat_insts = cat_insts
def __len__(self):
return len(self.images)
 def _load_data(self, idx):
cache_loaded = False
if idx in self.cache.keys() :
image = self.cache[idx]
cache_loaded = True
else:
if self.istest:
fpath = str(self.root/'test_images'/f'{self.images[idx]}.tiff')
 else:
fpath = str(self.root/'train_images'/f'{self.images[idx]}.tiff')
 image = skimage.io.MultiImage(fpath)[self.img_size]
if self.use_cache and not self.separate_image:
self.cache[idx] = image
if self.transform:
if self.separate_image:
if cache_loaded:
pass
else:
assert 'tile' in self.transform.keys()
image = self.transform['tile'](image=image)['image']
if self.use_cache:
self.cache[idx] = image
output = []
for tile in image:
output.append(self.transform['augmentation'](image=tile)['image'])
 output = torch.stack(output)
 else:
output = self.transform(image=image)['image']
label = self.labels[idx]
if self.insts is not None and self.cat_insts:
output = torch.flatten(output)
 insts = self.insts[idx]
if insts == 'karolinska':
insts = torch.tensor([0.0])
 elif insts == 'radboud':
insts = torch.tensor([1.0])
 output = torch.cat(( output, insts))
return output, label
def __getitem__(self, idx):
image, label = self._load_data(idx)
 if self.mixup:
idx2 = np.random.randint(0, len(self.images))
lam = np.random.beta(self.mixup_alpha, self.mixup_alpha)
 image2, label2 = self._load_data(idx2)
 image = torch.cat([torch.Tensor([lam]), image.view(-1), image2.view(-1)])
 label = lam * label +(1 - lam)* label2
if self.bin_label:
if self.mixup:
label_dec = label - label_int
label2 = torch.zeros(5)
 label2[:label_int] = 1.0
if label_int < 5:
label2[label_int] = label_dec
else:
label2 = torch.zeros(5)
 label2[:label] = 1
else:
label2 = label
if self.return_index:
return image, label2, idx
else:
return image, label2
thresholds = np.linspace(0, 1, 100)[1:]
scores_train = []
scores_val = []
scores_test = []
for t in thresholds:
s_train = matthews_corrcoef(
 y_train.values.astype(np.float),
yp_train > t 
)
 s_val = matthews_corrcoef(
 y_train.values.astype(np.float),
yp_val > t 
)
 s_test = matthews_corrcoef(
 y_train.values.astype(np.float),
yp_test > t 
)
 scores_train.append(s_train)
 scores_val.append(s_val)
 scores_test.append(s_test)
 plt.plot(thresholds, scores_train)
 plt.plot(thresholds, scores_val)
 plt.plot(thresholds, scores_test)
 plt.axvline(best_proba)
 plt.axvline(thresholds[np.argmax(scores_val)], ls='--')
 plt.show()
print(np.max(scores_val), thresholds[np.argmax(scores_val)] )<data_type_conversions>
def seed_everything(seed=2020):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 def print_config(cfg):
items = [
'name',
'patch_size', 'patch_dim',
'resume', 'img_size', 'batch_size', 'lr', 'epochs', 'CV', 'seed',
'use_cache', 'separate_image', 'return_index', 'bin_label', 'mixup',
'model', 'criterion', 'metric', 'log_metrics', 'stopper', 'event', 'transform',
]
print(f'
----- Config -----')
 for key in items:
try:
value = eval(f'cfg.{key}')
 print(f'{key}: {value}')
 except:
print(f'{key}: ERROR')
 print(f'----- Config -----
' )
train_pred['probability_thresholded'] =(train_pred['prediction'] > best_proba ).astype(np.int )<save_to_csv>
MAX_GRIDS = 200
def encode_gleason(gleason):
if gleason == 'negative':
a = b = 0
else:
a, b = gleason.split('+')
 a, b = int(a), int(b)
 if a >= 3:
a -= 1
else:
a = 1
if b >= 3:
b -= 1
else:
b = 1
a = [1 for _ in range(a)] + [0 for _ in range(4 - a)]
b = [1 for _ in range(b)] + [0 for _ in range(4 - b)]
return a + b
def Rotate(img, degrees=45): 
(h, w)= img.shape[:2]
M = cv2.getRotationMatrix2D(( w // 2, h // 2), degrees, 1.0)
 M_ = M[:, :2].T
corners = np.array([[w // 2, w // 2], [h // 2, -h // 2]])
 new_corners = M_ @ corners
del_y = max(np.abs(new_corners[0])) - w // 2
del_x = max(np.abs(new_corners[1])) - h // 2
pad_x, pad_y = int(max(del_x, 0)) , int(max(del_y, 0))
img = np.pad(img,(( pad_x, pad_x),(pad_y, pad_y),(0, 0)) , constant_values=255)
(h, w)= img.shape[:2]
M = cv2.getRotationMatrix2D(( w // 2, h // 2), degrees, 1.0)
 rotated = cv2.warpAffine(img, M,(w, h), cv2.INTER_LINEAR, borderMode=cv2.BORDER_CONSTANT,
borderValue=(255, 255, 255))
return rotated
def get_img(png_path, mode = 0, tile_size = 224, patch_dim = 8):
if '.png' in png_path:
img_ = cv2.imread(png_path)
 else:
img_ = io.MultiImage(png_path)[1]
img = cv2.resize(img_,(img_.shape[1], img_.shape[0]), cv2.INTER_AREA)
 mean = 1 - torch.tensor([.485,.456,.406] ).float().unsqueeze(1 ).unsqueeze(1 ).unsqueeze(0)
 std = torch.tensor([.229,.224,.225] ).float().unsqueeze(1 ).unsqueeze(1 ).unsqueeze(0)
 if mode % 2 == 1:
img = np.pad(img,(( tile_size // 2, tile_size // 2), 
(tile_size // 2, tile_size // 2), 
(0, 0)) , constant_values=255)
 degrees =(mode % 3)* 30
img = Rotate(img, degrees)
 size = tile_size
if img.shape[0] % size == 0:
pad_h = 0
else:
pad_h =(img.shape[0] // size + 1)* size - img.shape[0]
if img.shape[1] % size == 0:
pad_w = 0
else:
pad_w =(img.shape[1] // size + 1)* size - img.shape[1]
img = np.pad(img,(( pad_h, 0),(pad_w, 0),(0, 0)) , constant_values=255)
 h_, w_ = img.shape[0] // size, img.shape[1] // size
imgs = img.reshape(h_, size, w_, size, 3 ).transpose(0, 2, 1, 3, 4 ).reshape(-1, size, size, 3)
 brightness = imgs.reshape(imgs.shape[0], -1 ).mean(1)
 indices = np.argsort(brightness)
 num_valid = np.array(brightness < 250 ).astype(np.uint8 ).sum()
indices = indices[:num_valid]
img = np.concatenate(imgs[indices], axis=1)
 imgs = np.split(img, img.shape[1] // tile_size, axis=1)
 drop =(np.random.rand(len(imgs)) < 0.0 ).astype(np.float32)
 brightness = np.array([tile.mean() for tile in imgs])
 brightness = drop * 255 +(1 - drop)* brightness
indices = np.argsort(brightness)[:patch_dim * patch_dim]
imgs = [imgs[i].astype(np.uint8)for i in indices]
if len(imgs)< patch_dim * patch_dim:
deficit = patch_dim * patch_dim - len(imgs)
 imgs = np.concatenate([imgs + [np.ones([tile_size , tile_size, 3])* 255 for _ in range(deficit)]], axis=1)
 img = 255 - np.stack([img for img in imgs], axis=0)
 img = torch.from_numpy(img ).permute(0, 3, 1, 2 ).float() / 255.
 img =(img - mean)/ std
return img
class PandaDataset_Inference(Dataset):
def __init__(self, df, tta_mode, folder,tile_size = 224, patch_dim = 8):
self.names = df.image_id.values
self.df = df
self.tta_mode = tta_mode
self.folder = folder
self.tile_size = tile_size
self.patch_dim = patch_dim
def __getitem__(self, idx):
tta_mode = random.randint(0,7)
 img = get_img(os.path.join(self.folder, self.names[idx] + '.tiff'), tta_mode, self.tile_size, self.patch_dim)
 return img,img
def __len__(self):
return len(self.names)
train_pred.to_csv('train_results.csv', index=False )<create_dataframe>
def to_mish(model):
for child_name, child in model.named_children() :
if isinstance(child,(nn.ReLU, Swish, MemoryEfficientSwish, Swish_timm)) or child_name == '_swish':
print(child_name)
 setattr(model, child_name, Mish())
 else:
to_mish(child )
importances = pd.DataFrame()
for fold_ in range(len(models)) :
model = models[fold_]
imp_df = pd.DataFrame()
imp_df['feature'] = X_train.columns
imp_df['gain'] = model.feature_importance('gain')
 imp_df['fold'] = fold_ + 1
importances = pd.concat([importances, imp_df], axis=0, sort=False )<prepare_output>
class Classification:
img_size = 1
patch_size = 224
patch_dim = 8
batch_size = 8
separate_image = True
return_index = False
bin_label = False
cat_insts = False
model = PatchPoolModel2(
 base_model = senet_mod(se_resnext50_32x4d, pretrained=USE_PRETRAINED),
patch_total=patch_dim**2, num_classes=6 
)
 transform = {
'test': {
'tile': Compose([
MakePatches(patch_size, patch_dim, concat=False, always_apply=True)
 ]),
'augmentation': Compose([
ShiftScaleRotate(scale_limit=0.0625, rotate_limit=15, p=0.5),
HorizontalFlip(p=0.5), VerticalFlip(p=0.5),
Normalize([0.910, 0.819, 0.878],
[0.363, 0.499, 0.404], always_apply=True),
ToTensor()
])
 },
}
class Classification2:
img_size = 1
patch_size = 224
patch_dim = 8
batch_size = 8
separate_image = True
return_index = False
bin_label = False
cat_insts = False
model = PatchPoolModel2(
 base_model = xception_mod(in_channel=3, num_classes=1000, pretrained=USE_PRETRAINED),
patch_total=patch_dim**2, num_classes=6 
)
 transform = {
'test': {
'tile': Compose([
MakePatches(patch_size, patch_dim, concat=False, always_apply=True)
 ]),
'augmentation': Compose([
ShiftScaleRotate(scale_limit=0.0625, rotate_limit=15, p=0.5),
HorizontalFlip(p=0.5), VerticalFlip(p=0.5),
Normalize([0.910, 0.819, 0.878],
[0.363, 0.499, 0.404], always_apply=True),
ToTensor()
])
 },
}
class Regression:
img_size = 1
patch_size = 224
patch_dim = 8
batch_size = 12
separate_image = True
return_index = False
bin_label = False
cat_insts = False
model = PatchPoolModel2(
 base_model=senet_mod(se_resnext50_32x4d, pretrained=USE_PRETRAINED),
patch_total=patch_dim**2, num_classes=1 
)
 transform = {
'test': {
'tile': Compose([
MakePatches(patch_size, patch_dim, concat=False, always_apply=True)
 ]),
'augmentation': Compose([
ShiftScaleRotate(scale_limit=0.0625, rotate_limit=15, p=0.5),
HorizontalFlip(p=0.5), VerticalFlip(p=0.5),
Normalize([0.910, 0.819, 0.878],
[0.363, 0.499, 0.404], always_apply=True),
ToTensor()
])
 },
}
class OrdinalRegressionBear:
img_size = 1
patch_size = 224
patch_dim = 8
batch_size = 6
separate_image = True
return_index = False
bin_label = True
cat_insts = False
model = PatchPoolModel2(
 base_model = senet_mod(se_resnext50_32x4d, pretrained=USE_PRETRAINED),
patch_total=patch_dim**2, num_classes=5 
)
 transform = {
'test': {
'tile': Compose([
MakePatches(patch_size, patch_dim, concat=False, always_apply=True)
 ]),
'augmentation': Compose([
ShiftScaleRotate(scale_limit=0.0625, rotate_limit=15, p=0.5),
HorizontalFlip(p=0.5), VerticalFlip(p=0.5),
Normalize([0.910, 0.819, 0.878],
[0.363, 0.499, 0.404], always_apply=True),
ToTensor()
])
 },
}
class OrdinalRegressionBear101:
img_size = 1
patch_size = 224
patch_dim = 8
batch_size = 6
separate_image = True
return_index = False
bin_label = True
cat_insts = False
model = PatchPoolModel2(
 base_model = senet_mod(se_resnext101_32x4d, pretrained=USE_PRETRAINED),
patch_total=patch_dim**2, num_classes=5 
)
 transform = {
'test': {
'tile': Compose([
MakePatches(patch_size, patch_dim, concat=False, always_apply=True)
 ]),
'augmentation': Compose([
ShiftScaleRotate(scale_limit=0.0625, rotate_limit=15, p=0.5),
HorizontalFlip(p=0.5), VerticalFlip(p=0.5),
Normalize([0.910, 0.819, 0.878],
[0.363, 0.499, 0.404], always_apply=True),
ToTensor()
])
 },
}
class OrdinalRegression0:
img_size = 1
patch_size = 256
patch_dim = 6
batch_size = 6
separate_image = True
return_index = False
bin_label = True
cat_insts = False
model = PatchPoolModel2(
 base_model=FeatureEfficientNet.from_name('efficientnet-b0'),
patch_total=patch_dim**2, num_classes=5 
)
 to_mish(model)
 transform = {
'test': {
'tile': Compose([
MakePatches(patch_size, patch_dim, concat=False, always_apply=True)
 ]),
'augmentation': Compose([
ShiftScaleRotate(scale_limit=0.0625, rotate_limit=15, p=0.5),
HorizontalFlip(p=0.5), VerticalFlip(p=0.5),
Normalize([0.910, 0.819, 0.878],
[0.363, 0.499, 0.404], always_apply=True),
ToTensor()
])
 },
}
class OrdinalRegression1:
img_size = 1
patch_size = 224
patch_dim = 8
batch_size = 6
separate_image = True
return_index = False
bin_label = True
cat_insts = False
model = PatchPoolModel2(
 base_model=FeatureEfficientNet.from_name('efficientnet-b0'),
patch_total=patch_dim**2, num_classes=5 
)
 to_mish(model)
 transform = {
'test': {
'tile': Compose([
MakePatches(patch_size, patch_dim, concat=False, always_apply=True)
 ]),
'augmentation': Compose([
ShiftScaleRotate(scale_limit=0.0625, rotate_limit=15, p=0.5),
HorizontalFlip(p=0.5), VerticalFlip(p=0.5),
Normalize([0.910, 0.819, 0.878],
[0.363, 0.499, 0.404], always_apply=True),
ToTensor()
])
 },
}
class OrdinalRegression2:
img_size = 1
patch_size = 192
patch_dim = 8
batch_size = 6
separate_image = True
return_index = False
bin_label = True
cat_insts = False
model = PatchPoolModel2(
 base_model=FeatureEfficientNet.from_name('efficientnet-b0'),
patch_total=patch_dim**2, num_classes=5 
)
 to_mish(model)
 transform = {
'test': {
'tile': Compose([
MakePatches(patch_size, patch_dim, concat=False, always_apply=True)
 ]),
'augmentation': Compose([
ShiftScaleRotate(scale_limit=0.0625, rotate_limit=15, p=0.5),
HorizontalFlip(p=0.5), VerticalFlip(p=0.5),
Normalize([0.910, 0.819, 0.878],
[0.363, 0.499, 0.404], always_apply=True),
ToTensor()
])
 },
}
class MixOrdinalRegression:
img_size = 1
patch_size = 224
patch_dim = 7
batch_size = 8
separate_image = True
return_index = False
bin_label = True
cat_insts = False
model = MixPatchPoolModel(
 base_model=FeatureEfficientNet.from_name('efficientnet-b0'),
patch_size=patch_size, patch_total=patch_dim**2, num_classes=5, 
)
 transform = {
'test': {
'tile': Compose([
MakePatches(patch_size, patch_dim, concat=False, always_apply=True)
 ]),
'augmentation': Compose([
ShiftScaleRotate(scale_limit=0.0625, rotate_limit=15, p=0.5),
HorizontalFlip(p=0.5), VerticalFlip(p=0.5),
Normalize([0.910, 0.819, 0.878],
[0.363, 0.499, 0.404], always_apply=True),
ToTensor()
])
 },
}
prediction_errors = y_train.copy().to_frame()
prediction_errors.columns = ['target']
prediction_errors['error'] = np.abs(y_train.values - yp_train)
 prediction_errors = prediction_errors.sort_values('error' )<predict_on_test>
INPUT_PATH = Path('.. /input/prostate-cancer-grade-assessment/')
 USE_PRETRAINED = False
%%time
yp_test = np.zeros(X_test.shape[0])
 for j in range(len(models)) :
model = models[j]
yp_test += model.predict(X_test.values)/len(models )<create_dataframe>
def simple_inference_(snapshots, df, cfg, tta, folder):
print(cfg.patch_size, cfg.patch_dim)
 images = df.image_id.values
insts = df.data_provider.values
labels = np.zeros(len(df), dtype=int)
 predictions = np.zeros(( tta, len(snapshots), len(df)) , dtype=np.float16)
 model = deepcopy(cfg.model)
 model.to(device)
 ds = PandaDataset_Inference(df, None, folder, cfg.patch_size, cfg.patch_dim)
 loader = D.DataLoader(ds, batch_size=cfg.batch_size, shuffle=False, num_workers=2)
 with torch.no_grad() :
for tta_i, tta_fold in enumerate(range(tta)) :
pred_fold = []
for idx,(inputs, _)in enumerate(tqdm(loader, total=len(loader), desc=f'tta{tta_i}')) :
pred_batch = np.zeros(( len(snapshots), len(inputs)) , dtype=np.float16)
 for fold_i, fold_snapshot in enumerate(snapshots):
model.load_state_dict(torch.jit.load(fold_snapshot, map_location='cpu' ).state_dict())
 model.eval()
inputs = inputs.to(device)
 outputs = model(inputs)
 if outputs.shape[1] == 6:
outputs = F.softmax(outputs, dim=1 ).cpu().detach().numpy()
outputs = np.dot(outputs, np.arange(6))
elif outputs.shape[1] == 7:
outputs = F.softmax(outputs[:, :6], dim=1 ).cpu().detach().numpy()
outputs = np.dot(outputs, np.arange(6))
elif outputs.shape[1] == 5:
outputs = outputs.sigmoid().sum(1 ).cpu().detach().numpy()
elif outputs.shape[1] == 1:
outputs = outputs.reshape(-1 ).cpu().detach().numpy()
pred_batch[fold_i] = outputs
pred_fold.append(pred_batch)
 predictions[tta_i] = np.concatenate(pred_fold, axis=1)
 return predictions
def simple_inference(snapshots, df, cfg, tta=1, folder=None):
images = df.image_id.values
insts = df.data_provider.values
labels = np.zeros(len(df), dtype=int)
 ds = PandaDataset(
 images=images, labels=labels, insts=insts,
img_size=cfg.img_size, transform=cfg.transform['test'],
return_index=cfg.return_index, bin_label=cfg.bin_label,
separate_image=cfg.separate_image, cat_insts=cfg.cat_insts,
root_path=INPUT_PATH, istest=IS_TEST)
 loader = D.DataLoader(ds, batch_size=cfg.batch_size, shuffle=False, num_workers=2)
 predictions = np.zeros(( tta, len(snapshots), len(df)) , dtype=np.float16)
 model = deepcopy(cfg.model)
 model.to(device)
 with torch.no_grad() :
for tta_i, tta_fold in enumerate(range(tta)) :
pred_fold = []
for idx,(inputs, _)in enumerate(tqdm(loader, total=len(loader), desc=f'tta{tta_i}')) :
pred_batch = np.zeros(( len(snapshots), len(inputs)) , dtype=np.float16)
 for fold_i, fold_snapshot in enumerate(snapshots):
load_snapshots_to_model(fold_snapshot, model=model)
 model.eval()
inputs = inputs.to(device)
 outputs = model(inputs)
 if outputs.shape[1] == 6:
outputs = F.softmax(outputs, dim=1 ).cpu().detach().numpy()
outputs = np.dot(outputs, np.arange(6))
elif outputs.shape[1] == 7:
outputs = F.softmax(outputs[:, :6], dim=1 ).cpu().detach().numpy()
outputs = np.dot(outputs, np.arange(6))
elif outputs.shape[1] == 5:
outputs = outputs.sigmoid().sum(1 ).cpu().detach().numpy()
elif outputs.shape[1] == 1:
outputs = outputs.reshape(-1 ).cpu().detach().numpy()
pred_batch[fold_i] = outputs
pred_fold.append(pred_batch)
 predictions[tta_i] = np.concatenate(pred_fold, axis=1)
 return predictions
test_submission = pd.DataFrame(
 yp_test,
index=X_test.index,
columns=['probability']
)<data_type_conversions>
train_df = pd.read_csv(INPUT_PATH/'train.csv')
 test_df = pd.read_csv(INPUT_PATH/'test.csv')
 device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 TTA = 2
if(INPUT_PATH/'test_images' ).exists() :
print('Inference')
 target = test_df
IS_TEST = True
folder = INPUT_PATH/'test_images'
else:
print('Debug')
 target = train_df.head(10)
 IS_TEST = False
TTA = 1
folder = INPUT_PATH/'train_images'
seed_everything(2020)
 target.head()
all_predictions_ = []
for c in submission_setting_:
all_predictions_.append(simple_inference_(c['snapshot_path'], target, c['config'], TTA, folder))
all_predictions = np.concatenate(all_predictions_)
 print(all_predictions.shape)
test_submission['probability_thresholded'] =(yp_test > best_proba ).astype(np.int )<merge>
def logits_to_predictions(pred):
return np.dot(F.softmax(torch.Tensor(pred), 1 ).numpy() , np.arange(6))
def soft_vote(pred):
return np.clip(np.mean(pred, axis=0 ).round() , 0, 5 ).astype(int)
 def hard_vote(pred):
return np.clip(stats.mode(pred.round().astype(int), axis=0)[0][0], 0, 5 ).astype(int )
test_submission = pd.merge(
 meta_test_df[['id_measurement', 'signal_id']],
test_submission,
on='id_measurement',
how='left'
)<save_to_csv>
predictions = np.zeros(len(target), dtype=int)
 print(all_predictions)
 predictions = soft_vote(all_predictions.mean(1))
test_submission.to_csv('test_results.csv', index=False )<load_from_csv>
labels_ = target.isup_grade.values
qwk = QWK(6)
 print(qwk(labels_, predictions))
print(predictions )
submission = pd.read_csv(data_dir + '/sample_submission.csv' )<prepare_x_and_y>
OPTIMIZE_QWK = True
HARD_VOTE = False
limit_loss = 1 - 1/12
mean_predictions = all_predictions.mean(1)
 hard_predictions = np.zeros_like(mean_predictions)
 print(mean_predictions.shape)
 print(hard_predictions.shape)
 if OPTIMIZE_QWK:
oof_all_list = []
for ic, c in enumerate(submission_setting_):
oof_list = []
print('============================================')
 for npy in c['oof_path']:
print(npy)
 oof = np.load(npy)
 oof_list.append(oof)
 oof_all_list.append(np.sum(oof_list, axis=0))
all_coefs = []
for i_oof, oof_fold in enumerate(oof_all_list):
print('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!')
 print(len(oof_all_list))
mask_fold = oof_fold.sum(axis=1)!= 0
print(mask_fold.shape)
 print(np.sum(mask_fold))
output_shape = oof_fold.shape
if output_shape[0] == 9555:
_train = pd.read_csv('.. /input/mixup-baseline-b0-224-8-train2/train2.csv')
 _train = _train.query('certain_flag2 == 1')
 labels = _train.isup_grade.values
insts = _train.data_provider.values
elif output_shape[0] == 8493:
_train = pd.read_csv('.. /input/panda-seed-gacha/gatcha_train_nr20.csv')
 _train = _train.query('certain_flag2 == 1')
 labels = _train.isup_grade.values
insts = _train.data_provider.values
elif output_shape[0] == 10086:
_train = pd.read_csv('.. /input/panda-seed-gacha/gatcha_train_nr5.csv')
 labels = _train.isup_grade.values
insts = _train.data_provider.values
elif output_shape[0] == 9024:
_train = pd.read_csv('.. /input/panda-seed-gacha/gatcha_train_nr15.csv')
 _train = _train.query('certain_flag2 == 1')
 labels = _train.isup_grade.values
insts = _train.data_provider.values
labels = df.isup_grade.values
insts = df.data_provider.values
elif output_shape[0] == 10615:
_train = pd.read_csv('.. /input/panda-seed-gacha/train.csv')
 _train = _train[_train.image_id.values != '3790f55cad63053e956fb73027179707'].reset_index(drop=True)
 labels = _train.isup_grade.values
insts = _train.data_provider.values
else:
_train = pd.read_csv('.. /input/panda-seed-gacha/train.csv')
 labels = _train.isup_grade.values
insts = _train.data_provider.values
if output_shape[1] == 5:
oof_fold = torch.from_numpy(oof_fold[mask_fold] ).float()
label_fold = torch.zeros_like(oof_fold)
 for i, t in enumerate(labels[mask_fold]):
if t == 0:
continue
label_fold[i, 0:t] = 1
criterion = nn.BCEWithLogitsLoss(reduction='none')
 loss_values = criterion(oof_fold, label_fold ).mean(1)
 _, certainty_mask = loss_values.topk(round(len(oof_fold)*limit_loss), largest=False)
 oof_to_use = oof_fold[certainty_mask].sigmoid().sum(1 ).numpy()
label_to_use = label_fold[certainty_mask].sum(1 ).numpy()
analyse_results(oof_to_use.round() , label_to_use)
 optR = OptimizedRounder()
optR.fit(oof_to_use, label_to_use)
 coefficients = optR.coefficients()
print(coefficients)
 all_coefs.append(coefficients)
 hard_predictions[i_oof] = optR.predict(mean_predictions[i_oof], coefficients)
 analyse_results(optR.predict(oof_to_use, coefficients), label_to_use)
 all_coefs = np.vstack(all_coefs)
 print(all_coefs)
 print(all_coefs.mean(0))
soft_predictions = optR.predict(mean_predictions.mean(0), all_coefs.mean(0))
if HARD_VOTE:
print('HARD_VOTE')
 print(hard_predictions)
 optimized_predictions = mode(hard_predictions)[0][0]
print(optimized_predictions)
 else:
print('SOFT_PRED')
 print(mean_predictions)
 print(soft_predictions)
 optimized_predictions = soft_predictions
else:
print(all_predictions.mean(0 ).mean(0))
optimized_predictions = all_predictions.mean(0 ).mean(0 ).round()
print(optimized_predictions )
assert np.all(submission['signal_id'].values == test_submission['signal_id'].values)
 submission['target'] = test_submission['probability_thresholded'].values<save_to_csv>
submission = pd.read_csv(INPUT_PATH/'sample_submission.csv')
 submission['isup_grade'] = optimized_predictions.astype(int)
 submission.head(20 )
<save_to_csv><EOS> 
if(INPUT_PATH/'test_images' ).exists() :
if submission['isup_grade'].sum() > 0:
submission.to_csv('submission.csv', index=False)
 else:
pass
else:
submission.to_csv('submission.csv', index=False )