Kaggle competition: Cornell Birdcall Identification
<SOS> metric: MeanFScoreBeta Kaggle data source: cornell-birdcall-identification<feature_engineering>
import math
import cv2
import audioread
import collections
import logging
import os
import random
import re
import time
import warnings
import librosa
import numpy as np
import pandas as pd
import soundfile as sf
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.utils.data as data
from contextlib import contextmanager
from functools import partial
from pathlib import Path
from typing import Optional
from fastprogress import progress_bar
from sklearn.metrics import f1_score
from torch.nn import Conv2d, Module, Linear, BatchNorm2d, ReLU
from torch.nn.modules.utils import _pair
<define_variables>
def set_seed(seed: int = 42):
random.seed(seed)
 np.random.seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
def get_logger(out_file=None):
logger = logging.getLogger()
formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s")
 logger.handlers = []
logger.setLevel(logging.INFO)
 handler = logging.StreamHandler()
handler.setFormatter(formatter)
 handler.setLevel(logging.INFO)
 logger.addHandler(handler)
 if out_file is not None:
fh = logging.FileHandler(out_file)
 fh.setFormatter(formatter)
 fh.setLevel(logging.INFO)
 logger.addHandler(fh)
 logger.info("logger set up")
 return logger
@contextmanager
def timer(name: str, logger: Optional[logging.Logger] = None):
t0 = time.time()
msg = f"[{name}] start"
if logger is None:
print(msg)
 else:
logger.info(msg)
 yield
msg = f"[{name}] done in {time.time() - t0:.2f} s"
if logger is None:
print(msg)
 else:
logger.info(msg )
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]<categorify>
logger = get_logger("main.log")
 set_seed(1213 )
for col in(GENES + CELLS):
transformer = QuantileTransformer(n_quantiles=100,random_state=0, output_distribution="normal")
 vec_len = len(train_features[col].values)
 vec_len_test = len(test_features[col].values)
 raw_vec = train_features[col].values.reshape(vec_len, 1)
 transformer.fit(raw_vec)
 train_features[col] = transformer.transform(raw_vec ).reshape(1, vec_len)[0]
test_features[col] = transformer.transform(test_features[col].values.reshape(vec_len_test, 1)).reshape(1, vec_len_test)[0]<set_options>
TARGET_SR = 32000
TEST = Path(".. /input/birdsong-recognition/test_audio" ).exists()
def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(seed=42 )<concatenate>
if TEST:
DATA_DIR = Path(".. /input/birdsong-recognition/")
 else:
DATA_DIR = Path(".. /input/birdcall-check/")
 test = pd.read_csv(DATA_DIR / "test.csv")
 test_audio = DATA_DIR / "test_audio"
test.head()
n_comp = 600
data = pd.concat([pd.DataFrame(train_features[GENES]), pd.DataFrame(test_features[GENES])])
 data2 =(PCA(n_components=n_comp, random_state=42 ).fit_transform(data[GENES]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 test2 = pd.DataFrame(test2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1 )<concatenate>
sub = pd.read_csv(".. /input/birdsong-recognition/sample_submission.csv")
 sub.to_csv("submission.csv", index=False )
n_comp = 50
data = pd.concat([pd.DataFrame(train_features[CELLS]), pd.DataFrame(test_features[CELLS])])
 data2 =(PCA(n_components=n_comp, random_state=42 ).fit_transform(data[CELLS]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 test2 = pd.DataFrame(test2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1 )<create_dataframe>
def init_layer(layer):
nn.init.xavier_uniform_(layer.weight)
 if hasattr(layer, "bias"):
if layer.bias is not None:
layer.bias.data.fill_(0.)
def init_bn(bn):
bn.bias.data.fill_(0.)
bn.weight.data.fill_(1.0)
 def interpolate(x: torch.Tensor, ratio: int):

(batch_size, time_steps, classes_num)= x.shape
upsampled = x[:, :, None, :].repeat(1, 1, ratio, 1)
 upsampled = upsampled.reshape(batch_size, time_steps * ratio, classes_num)
 return upsampled
def pad_framewise_output(framewise_output: torch.Tensor, frames_num: int):

 pad = framewise_output[:, -1:, :].repeat(
 1, frames_num - framewise_output.shape[1], 1)

output = torch.cat(( framewise_output, pad), dim=1)

return output
class ConvBlock(nn.Module):
def __init__(self, in_channels: int, out_channels: int):
super().__init__()
self.conv1 = nn.Conv2d(
 in_channels=in_channels,
out_channels=out_channels,
kernel_size=(3, 3),
stride=(1, 1),
padding=(1, 1),
bias=False)
 self.conv2 = nn.Conv2d(
 in_channels=out_channels,
out_channels=out_channels,
kernel_size=(3, 3),
stride=(1, 1),
padding=(1, 1),
bias=False)
 self.bn1 = nn.BatchNorm2d(out_channels)
 self.bn2 = nn.BatchNorm2d(out_channels)
 self.init_weight()
def init_weight(self):
init_layer(self.conv1)
 init_layer(self.conv2)
 init_bn(self.bn1)
 init_bn(self.bn2)
 def forward(self, input, pool_size=(2, 2), pool_type='avg'):
x = input
x = F.relu_(self.bn1(self.conv1(x)))
 x = F.relu_(self.bn2(self.conv2(x)))
 if pool_type == 'max':
x = F.max_pool2d(x, kernel_size=pool_size)
 elif pool_type == 'avg':
x = F.avg_pool2d(x, kernel_size=pool_size)
 elif pool_type == 'avg+max':
x1 = F.avg_pool2d(x, kernel_size=pool_size)
 x2 = F.max_pool2d(x, kernel_size=pool_size)
 x = x1 + x2
else:
raise Exception('Incorrect argument!')
 return x
class AttBlock(nn.Module):
def __init__(self,
in_features: int,
out_features: int,
activation="linear",
temperature=1.0):
super().__init__()
self.activation = activation
self.temperature = temperature
self.att = nn.Conv1d(
 in_channels=in_features,
out_channels=out_features,
kernel_size=1,
stride=1,
padding=0,
bias=True)
 self.cla = nn.Conv1d(
 in_channels=in_features,
out_channels=out_features,
kernel_size=1,
stride=1,
padding=0,
bias=True)
 self.init_weights()
def init_weights(self):
init_layer(self.att)
 def forward(self, x):
norm_att = torch.softmax(torch.tanh(self.att(x)) , dim=-1)
 cla = self.nonlinear_transform(self.cla(x))
x = torch.sum(norm_att * cla, dim=2)
 return x, norm_att, cla
def nonlinear_transform(self, x):
if self.activation == 'linear':
return x
elif self.activation == 'sigmoid':
return torch.sigmoid(x )
var_thresh = VarianceThreshold(0.8)
 data = train_features.append(test_features)
 data_transformed = var_thresh.fit_transform(data.iloc[:, 4:])
 train_features_transformed = data_transformed[ : train_features.shape[0]]
test_features_transformed = data_transformed[-test_features.shape[0] : ]
train_features = pd.DataFrame(train_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 train_features = pd.concat([train_features, pd.DataFrame(train_features_transformed)], axis=1)
 test_features = pd.DataFrame(test_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 test_features = pd.concat([test_features, pd.DataFrame(test_features_transformed)], axis=1)
 train_features.shape 
<merge>
class SplAtConv2d(Module):

 def __init__(self, in_channels, channels, kernel_size, stride=(1, 1), padding=(0, 0),
dilation=(1, 1), groups=1, bias=True,
radix=2, reduction_factor=4,
rectify=False, rectify_avg=False, norm_layer=None,
dropblock_prob=0.0, **kwargs):
super(SplAtConv2d, self ).__init__()
padding = _pair(padding)
 self.rectify = rectify and(padding[0] > 0 or padding[1] > 0)
 self.rectify_avg = rectify_avg
inter_channels = max(in_channels*radix//reduction_factor, 32)
 self.radix = radix
self.cardinality = groups
self.channels = channels
self.dropblock_prob = dropblock_prob
if self.rectify:
self.conv = RFConv2d(in_channels, channels*radix, kernel_size, stride, padding, dilation,
groups=groups*radix, bias=bias, average_mode=rectify_avg, **kwargs)
 else:
self.conv = Conv2d(in_channels, channels*radix, kernel_size, stride, padding, dilation,
groups=groups*radix, bias=bias, **kwargs)
 self.use_bn = norm_layer is not None
if self.use_bn:
self.bn0 = norm_layer(channels*radix)
 self.relu = ReLU(inplace=True)
 self.fc1 = Conv2d(channels, inter_channels, 1, groups=self.cardinality)
 if self.use_bn:
self.bn1 = norm_layer(inter_channels)
 self.fc2 = Conv2d(inter_channels, channels*radix, 1, groups=self.cardinality)
 if dropblock_prob > 0.0:
self.dropblock = DropBlock2D(dropblock_prob, 3)
 self.rsoftmax = rSoftMax(radix, groups)
 def forward(self, x):
x = self.conv(x)
 if self.use_bn:
x = self.bn0(x)
 if self.dropblock_prob > 0.0:
x = self.dropblock(x)
 x = self.relu(x)
 batch, rchannel = x.shape[:2]
if self.radix > 1:
if torch.__version__ < '1.5':
splited = torch.split(x, int(rchannel//self.radix), dim=1)
 else:
splited = torch.split(x, rchannel//self.radix, dim=1)
 gap = sum(splited)
 else:
gap = x
gap = F.adaptive_avg_pool2d(gap, 1)
 gap = self.fc1(gap)
 if self.use_bn:
gap = self.bn1(gap)
 gap = self.relu(gap)
 atten = self.fc2(gap)
 atten = self.rsoftmax(atten ).view(batch, -1, 1, 1)
 if self.radix > 1:
if torch.__version__ < '1.5':
attens = torch.split(atten, int(rchannel//self.radix), dim=1)
 else:
attens = torch.split(atten, rchannel//self.radix, dim=1)
 out = sum([att*split for(att, split)in zip(attens, splited)])
 else:
out = atten * x
return out.contiguous()
class rSoftMax(nn.Module):
def __init__(self, radix, cardinality):
super().__init__()
self.radix = radix
self.cardinality = cardinality
def forward(self, x):
batch = x.size(0)
 if self.radix > 1:
x = x.view(batch, self.cardinality, self.radix, -1 ).transpose(1, 2)
 x = F.softmax(x, dim=1)
 x = x.reshape(batch, -1)
 else:
x = torch.sigmoid(x)
 return x
class DropBlock2D(object):
def __init__(self, *args, **kwargs):
raise NotImplementedError
class GlobalAvgPool2d(nn.Module):
def __init__(self):

 super(GlobalAvgPool2d, self ).__init__()
def forward(self, inputs):
return nn.functional.adaptive_avg_pool2d(inputs, 1 ).view(inputs.size(0), -1)
 class Bottleneck(nn.Module):

 expansion = 4
def __init__(self, inplanes, planes, stride=1, downsample=None,
radix=1, cardinality=1, bottleneck_width=64,
avd=False, avd_first=False, dilation=1, is_first=False,
rectified_conv=False, rectify_avg=False,
norm_layer=None, dropblock_prob=0.0, last_gamma=False):
super(Bottleneck, self ).__init__()
group_width = int(planes *(bottleneck_width / 64.))* cardinality
self.conv1 = nn.Conv2d(inplanes, group_width, kernel_size=1, bias=False)
 self.bn1 = norm_layer(group_width)
 self.dropblock_prob = dropblock_prob
self.radix = radix
self.avd = avd and(stride > 1 or is_first)
 self.avd_first = avd_first
if self.avd:
self.avd_layer = nn.AvgPool2d(3, stride, padding=1)
 stride = 1
if dropblock_prob > 0.0:
self.dropblock1 = DropBlock2D(dropblock_prob, 3)
 if radix == 1:
self.dropblock2 = DropBlock2D(dropblock_prob, 3)
 self.dropblock3 = DropBlock2D(dropblock_prob, 3)
 if radix >= 1:
self.conv2 = SplAtConv2d(
 group_width, group_width, kernel_size=3,
stride=stride, padding=dilation,
dilation=dilation, groups=cardinality, bias=False,
radix=radix, rectify=rectified_conv,
rectify_avg=rectify_avg,
norm_layer=norm_layer,
dropblock_prob=dropblock_prob)
 elif rectified_conv:
self.conv2 = RFConv2d(
 group_width, group_width, kernel_size=3, stride=stride,
padding=dilation, dilation=dilation,
groups=cardinality, bias=False,
average_mode=rectify_avg)
 self.bn2 = norm_layer(group_width)
 else:
self.conv2 = nn.Conv2d(
 group_width, group_width, kernel_size=3, stride=stride,
padding=dilation, dilation=dilation,
groups=cardinality, bias=False)
 self.bn2 = norm_layer(group_width)
 self.conv3 = nn.Conv2d(
 group_width, planes * 4, kernel_size=1, bias=False)
 self.bn3 = norm_layer(planes*4)
 if last_gamma:
zeros_(self.bn3.weight)
 self.relu = nn.ReLU(inplace=True)
 self.downsample = downsample
self.dilation = dilation
self.stride = stride
def forward(self, x):
residual = x
out = self.conv1(x)
 out = self.bn1(out)
 if self.dropblock_prob > 0.0:
out = self.dropblock1(out)
 out = self.relu(out)
 if self.avd and self.avd_first:
out = self.avd_layer(out)
 out = self.conv2(out)
 if self.radix == 0:
out = self.bn2(out)
 if self.dropblock_prob > 0.0:
out = self.dropblock2(out)
 out = self.relu(out)
 if self.avd and not self.avd_first:
out = self.avd_layer(out)
 out = self.conv3(out)
 out = self.bn3(out)
 if self.dropblock_prob > 0.0:
out = self.dropblock3(out)
 if self.downsample is not None:
residual = self.downsample(x)
 out += residual
out = self.relu(out)
 return out
train = train_features.merge(train_targets_scored, on='sig_id')
 train = train[train['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 test = test_features[test_features['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 target = train[train_targets_scored.columns]<drop_column>
class ResNest(nn.Module):

 def __init__(self, block, layers, radix=1, groups=1, bottleneck_width=64,
num_classes=1000, dilated=False, dilation=1,
deep_stem=False, stem_width=64, avg_down=False,
rectified_conv=False, rectify_avg=False,
avd=False, avd_first=False,
final_drop=0.0, dropblock_prob=0,
last_gamma=False, norm_layer=nn.BatchNorm2d):
self.cardinality = groups
self.bottleneck_width = bottleneck_width
self.inplanes = stem_width*2 if deep_stem else 64
self.avg_down = avg_down
self.last_gamma = last_gamma
self.radix = radix
self.avd = avd
self.avd_first = avd_first
super(ResNest, self ).__init__()
self.rectified_conv = rectified_conv
self.rectify_avg = rectify_avg
if rectified_conv:
conv_layer = RFConv2d
else:
conv_layer = nn.Conv2d
conv_kwargs = {'average_mode': rectify_avg} if rectified_conv else {}
if deep_stem:
self.conv1 = nn.Sequential(
 conv_layer(3, stem_width, kernel_size=3, stride=2, padding=1, bias=False, **conv_kwargs),
norm_layer(stem_width),
nn.ReLU(inplace=True),
conv_layer(stem_width, stem_width, kernel_size=3, stride=1, padding=1, bias=False, **conv_kwargs),
norm_layer(stem_width),
nn.ReLU(inplace=True),
conv_layer(stem_width, stem_width*2, kernel_size=3, stride=1, padding=1, bias=False, **conv_kwargs), 
)
 else:
self.conv1 = conv_layer(3, 64, kernel_size=7, stride=2, padding=3,
bias=False, **conv_kwargs)
 self.bn1 = norm_layer(self.inplanes)
 self.relu = nn.ReLU(inplace=True)
 self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
 self.layer1 = self._make_layer(block, 64, layers[0], norm_layer=norm_layer, is_first=False)
 self.layer2 = self._make_layer(block, 128, layers[1], stride=2, norm_layer=norm_layer)
 if dilated or dilation == 4:
self.layer3 = self._make_layer(block, 256, layers[2], stride=1,
dilation=2, norm_layer=norm_layer,
dropblock_prob=dropblock_prob)
 self.layer4 = self._make_layer(block, 512, layers[3], stride=1,
dilation=4, norm_layer=norm_layer,
dropblock_prob=dropblock_prob)
 elif dilation==2:
self.layer3 = self._make_layer(block, 256, layers[2], stride=2,
dilation=1, norm_layer=norm_layer,
dropblock_prob=dropblock_prob)
 self.layer4 = self._make_layer(block, 512, layers[3], stride=1,
dilation=2, norm_layer=norm_layer,
dropblock_prob=dropblock_prob)
 else:
self.layer3 = self._make_layer(block, 256, layers[2], stride=2,
norm_layer=norm_layer,
dropblock_prob=dropblock_prob)
 self.layer4 = self._make_layer(block, 512, layers[3], stride=2,
norm_layer=norm_layer,
dropblock_prob=dropblock_prob)
 self.avgpool = GlobalAvgPool2d()
self.drop = nn.Dropout(final_drop)if final_drop > 0.0 else None
self.fc = nn.Linear(512 * block.expansion, num_classes)
 for m in self.modules() :
if isinstance(m, nn.Conv2d):
n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
m.weight.data.normal_(0, math.sqrt(2./ n))
elif isinstance(m, norm_layer):
m.weight.data.fill_(1)
 m.bias.data.zero_()
def _make_layer(self, block, planes, blocks, stride=1, dilation=1, norm_layer=None,
dropblock_prob=0.0, is_first=True):
downsample = None
if stride != 1 or self.inplanes != planes * block.expansion:
down_layers = []
if self.avg_down:
if dilation == 1:
down_layers.append(nn.AvgPool2d(kernel_size=stride, stride=stride,
ceil_mode=True, count_include_pad=False))
else:
down_layers.append(nn.AvgPool2d(kernel_size=1, stride=1,
ceil_mode=True, count_include_pad=False))
down_layers.append(nn.Conv2d(self.inplanes, planes * block.expansion,
kernel_size=1, stride=1, bias=False))
else:
down_layers.append(nn.Conv2d(self.inplanes, planes * block.expansion,
kernel_size=1, stride=stride, bias=False))
down_layers.append(norm_layer(planes * block.expansion))
downsample = nn.Sequential(*down_layers)
 layers = []
if dilation == 1 or dilation == 2:
layers.append(block(self.inplanes, planes, stride, downsample=downsample,
radix=self.radix, cardinality=self.cardinality,
bottleneck_width=self.bottleneck_width,
avd=self.avd, avd_first=self.avd_first,
dilation=1, is_first=is_first, rectified_conv=self.rectified_conv,
rectify_avg=self.rectify_avg,
norm_layer=norm_layer, dropblock_prob=dropblock_prob,
last_gamma=self.last_gamma))
elif dilation == 4:
layers.append(block(self.inplanes, planes, stride, downsample=downsample,
radix=self.radix, cardinality=self.cardinality,
bottleneck_width=self.bottleneck_width,
avd=self.avd, avd_first=self.avd_first,
dilation=2, is_first=is_first, rectified_conv=self.rectified_conv,
rectify_avg=self.rectify_avg,
norm_layer=norm_layer, dropblock_prob=dropblock_prob,
last_gamma=self.last_gamma))
else:
raise RuntimeError("=> unknown dilation size: {}".format(dilation))
self.inplanes = planes * block.expansion
for i in range(1, blocks):
layers.append(block(self.inplanes, planes,
radix=self.radix, cardinality=self.cardinality,
bottleneck_width=self.bottleneck_width,
avd=self.avd, avd_first=self.avd_first,
dilation=dilation, rectified_conv=self.rectified_conv,
rectify_avg=self.rectify_avg,
norm_layer=norm_layer, dropblock_prob=dropblock_prob,
last_gamma=self.last_gamma))
return nn.Sequential(*layers)
 def forward(self, x):
x = self.conv1(x)
 x = self.bn1(x)
 x = self.relu(x)
 x = self.maxpool(x)
 x = self.layer1(x)
 x = self.layer2(x)
 x = self.layer3(x)
 x = self.layer4(x)
 x = self.avgpool(x)
 x = torch.flatten(x, 1)
 if self.drop:
x = self.drop(x)
 x = self.fc(x)
 multilabel_proba = torch.sigmoid(x)
 multiclass_proba = torch.softmax(x, dim=1)
 return {
"logits": x,
"multilabel_proba": multilabel_proba,
"multiclass_proba": multiclass_proba
}
train = train.drop('cp_type', axis=1)
 test = test.drop('cp_type', axis=1 )<drop_column>
class Swish(nn.Module):
def forward(self, x):
return x * torch.sigmoid(x)
 class SwishImplementation(torch.autograd.Function):
@staticmethod
def forward(ctx, i):
result = i * torch.sigmoid(i)
 ctx.save_for_backward(i)
 return result
@staticmethod
def backward(ctx, grad_output):
i = ctx.saved_tensors[0]
sigmoid_i = torch.sigmoid(i)
 return grad_output *(sigmoid_i *(1 + i *(1 - sigmoid_i)))
 class MemoryEfficientSwish(nn.Module):
def forward(self, x):
return SwishImplementation.apply(x)
 def round_filters(filters, global_params):

 multiplier = global_params.width_coefficient
if not multiplier:
return filters
divisor = global_params.depth_divisor
min_depth = global_params.min_depth
filters *= multiplier
min_depth = min_depth or divisor
new_filters = max(min_depth, int(filters + divisor / 2)// divisor * divisor)
 if new_filters < 0.9 * filters:
new_filters += divisor
return int(new_filters)
 def round_repeats(repeats, global_params):

 multiplier = global_params.depth_coefficient
if not multiplier:
return repeats
return int(math.ceil(multiplier * repeats))
def drop_connect(inputs, p, training):

 assert p >= 0 and p <= 1, 'p must be in range of [0,1]'
if not training:
return inputs
batch_size = inputs.shape[0]
keep_prob = 1 - p
random_tensor = keep_prob
random_tensor += torch.rand([batch_size, 1, 1, 1], dtype=inputs.dtype, device=inputs.device)
 binary_tensor = torch.floor(random_tensor)
 output = inputs / keep_prob * binary_tensor
return output
def get_width_and_height_from_size(x):

 if isinstance(x, int):
return x, x
if isinstance(x, list)or isinstance(x, tuple):
return x
else:
raise TypeError()
def calculate_output_image_size(input_image_size, stride):

 if input_image_size is None:
return None
image_height, image_width = get_width_and_height_from_size(input_image_size)
 stride = stride if isinstance(stride, int)else stride[0]
image_height = int(math.ceil(image_height / stride))
image_width = int(math.ceil(image_width / stride))
return [image_height, image_width]
def get_same_padding_conv2d(image_size=None):

 if image_size is None:
return Conv2dDynamicSamePadding
else:
return partial(Conv2dStaticSamePadding, image_size=image_size)
 class Conv2dDynamicSamePadding(nn.Conv2d):

 def __init__(self, in_channels, out_channels, kernel_size, stride=1, dilation=1, groups=1, bias=True):
super().__init__(in_channels, out_channels, kernel_size, stride, 0, dilation, groups, bias)
 self.stride = self.stride if len(self.stride)== 2 else [self.stride[0]] * 2
def forward(self, x):
ih, iw = x.size() [-2:]
kh, kw = self.weight.size() [-2:]
sh, sw = self.stride
oh, ow = math.ceil(ih / sh), math.ceil(iw / sw)
 pad_h = max(( oh - 1)* self.stride[0] +(kh - 1)* self.dilation[0] + 1 - ih, 0)
 pad_w = max(( ow - 1)* self.stride[1] +(kw - 1)* self.dilation[1] + 1 - iw, 0)
 if pad_h > 0 or pad_w > 0:
x = F.pad(x, [pad_w // 2, pad_w - pad_w // 2, pad_h // 2, pad_h - pad_h // 2])
 return F.conv2d(x, self.weight, self.bias, self.stride, self.padding, self.dilation, self.groups)
 class Conv2dStaticSamePadding(nn.Conv2d):

 def __init__(self, in_channels, out_channels, kernel_size, stride=1, image_size=None, **kwargs):
super().__init__(in_channels, out_channels, kernel_size, stride, **kwargs)
 self.stride = self.stride if len(self.stride)== 2 else [self.stride[0]] * 2
assert image_size is not None
ih, iw =(image_size, image_size)if isinstance(image_size, int)else image_size
kh, kw = self.weight.size() [-2:]
sh, sw = self.stride
oh, ow = math.ceil(ih / sh), math.ceil(iw / sw)
 pad_h = max(( oh - 1)* self.stride[0] +(kh - 1)* self.dilation[0] + 1 - ih, 0)
 pad_w = max(( ow - 1)* self.stride[1] +(kw - 1)* self.dilation[1] + 1 - iw, 0)
 if pad_h > 0 or pad_w > 0:
self.static_padding = nn.ZeroPad2d(( pad_w // 2, pad_w - pad_w // 2, pad_h // 2, pad_h - pad_h // 2))
else:
self.static_padding = Identity()
def forward(self, x):
x = self.static_padding(x)
 x = F.conv2d(x, self.weight, self.bias, self.stride, self.padding, self.dilation, self.groups)
 return x
def get_same_padding_maxPool2d(image_size=None):

 if image_size is None:
return MaxPool2dDynamicSamePadding
else:
return partial(MaxPool2dStaticSamePadding, image_size=image_size)
 class MaxPool2dDynamicSamePadding(nn.MaxPool2d):

 def __init__(self, kernel_size, stride, padding=0, dilation=1, return_indices=False, ceil_mode=False):
super().__init__(kernel_size, stride, padding, dilation, return_indices, ceil_mode)
 self.stride = [self.stride] * 2 if isinstance(self.stride, int)else self.stride
self.kernel_size = [self.kernel_size] * 2 if isinstance(self.kernel_size, int)else self.kernel_size
self.dilation = [self.dilation] * 2 if isinstance(self.dilation, int)else self.dilation
def forward(self, x):
ih, iw = x.size() [-2:]
kh, kw = self.kernel_size
sh, sw = self.stride
oh, ow = math.ceil(ih / sh), math.ceil(iw / sw)
 pad_h = max(( oh - 1)* self.stride[0] +(kh - 1)* self.dilation[0] + 1 - ih, 0)
 pad_w = max(( ow - 1)* self.stride[1] +(kw - 1)* self.dilation[1] + 1 - iw, 0)
 if pad_h > 0 or pad_w > 0:
x = F.pad(x, [pad_w // 2, pad_w - pad_w // 2, pad_h // 2, pad_h - pad_h // 2])
 return F.max_pool2d(x, self.kernel_size, self.stride, self.padding,
self.dilation, self.ceil_mode, self.return_indices)
 class MaxPool2dStaticSamePadding(nn.MaxPool2d):

 def __init__(self, kernel_size, stride, image_size=None, **kwargs):
super().__init__(kernel_size, stride, **kwargs)
 self.stride = [self.stride] * 2 if isinstance(self.stride, int)else self.stride
self.kernel_size = [self.kernel_size] * 2 if isinstance(self.kernel_size, int)else self.kernel_size
self.dilation = [self.dilation] * 2 if isinstance(self.dilation, int)else self.dilation
assert image_size is not None
ih, iw =(image_size, image_size)if isinstance(image_size, int)else image_size
kh, kw = self.kernel_size
sh, sw = self.stride
oh, ow = math.ceil(ih / sh), math.ceil(iw / sw)
 pad_h = max(( oh - 1)* self.stride[0] +(kh - 1)* self.dilation[0] + 1 - ih, 0)
 pad_w = max(( ow - 1)* self.stride[1] +(kw - 1)* self.dilation[1] + 1 - iw, 0)
 if pad_h > 0 or pad_w > 0:
self.static_padding = nn.ZeroPad2d(( pad_w // 2, pad_w - pad_w // 2, pad_h // 2, pad_h - pad_h // 2))
else:
self.static_padding = Identity()
def forward(self, x):
x = self.static_padding(x)
 x = F.max_pool2d(x, self.kernel_size, self.stride, self.padding,
self.dilation, self.ceil_mode, self.return_indices)
 return x
class Identity(nn.Module):

 def __init__(self):
super(Identity, self ).__init__()
def forward(self, input):
return input
def efficientnet_params(model_name):

 params_dict = {
'efficientnet-b0':(1.0, 1.0, 224, 0.2),
'efficientnet-b1':(1.0, 1.1, 240, 0.2),
'efficientnet-b2':(1.1, 1.2, 260, 0.3),
'efficientnet-b3':(1.2, 1.4, 300, 0.3),
'efficientnet-b4':(1.4, 1.8, 380, 0.4),
'efficientnet-b5':(1.6, 2.2, 456, 0.4),
'efficientnet-b6':(1.8, 2.6, 528, 0.5),
'efficientnet-b7':(2.0, 3.1, 600, 0.5),
'efficientnet-b8':(2.2, 3.6, 672, 0.5),
'efficientnet-l2':(4.3, 5.3, 800, 0.5),
}
return params_dict[model_name]
def get_model_params(model_name, override_params):

 if model_name.startswith('efficientnet'):
w, d, s, p = efficientnet_params(model_name)
 blocks_args, global_params = efficientnet(
 width_coefficient=w, depth_coefficient=d, dropout_rate=p, image_size=s)
 else:
raise NotImplementedError('model name is not pre-defined: %s' % model_name)
 if override_params:
global_params = global_params._replace(**override_params)
 return blocks_args, global_params
class BlockDecoder(object):

 @staticmethod
def _decode_block_string(block_string):

 assert isinstance(block_string, str)
 ops = block_string.split('_')
 options = {}
for op in ops:
splits = re.split(r'(\d.*)', op)
 if len(splits)>= 2:
key, value = splits[:2]
options[key] = value
assert(( 's' in options and len(options['s'])== 1)or 
(len(options['s'])== 2 and options['s'][0] == options['s'][1]))
return BlockArgs(
 num_repeat=int(options['r']),
kernel_size=int(options['k']),
stride=[int(options['s'][0])],
expand_ratio=int(options['e']),
input_filters=int(options['i']),
output_filters=int(options['o']),
se_ratio=float(options['se'])if 'se' in options else None,
id_skip=('noskip' not in block_string))
@staticmethod
def _encode_block_string(block):

 args = [
'r%d' % block.num_repeat,
'k%d' % block.kernel_size,
's%d%d' %(block.strides[0], block.strides[1]),
'e%s' % block.expand_ratio,
'i%d' % block.input_filters,
'o%d' % block.output_filters
]
if 0 < block.se_ratio <= 1:
args.append('se%s' % block.se_ratio)
 if block.id_skip is False:
args.append('noskip')
 return '_'.join(args)
 @staticmethod
def decode(string_list):

 assert isinstance(string_list, list)
 blocks_args = []
for block_string in string_list:
blocks_args.append(BlockDecoder._decode_block_string(block_string))
return blocks_args
@staticmethod
def encode(blocks_args):

 block_strings = []
for block in blocks_args:
block_strings.append(BlockDecoder._encode_block_string(block))
return block_strings
def efficientnet(width_coefficient=None, depth_coefficient=None, image_size=None,
dropout_rate=0.2, drop_connect_rate=0.2, num_classes=1000):

 blocks_args = [
'r1_k3_s11_e1_i32_o16_se0.25',
'r2_k3_s22_e6_i16_o24_se0.25',
'r2_k5_s22_e6_i24_o40_se0.25',
'r3_k3_s22_e6_i40_o80_se0.25',
'r3_k5_s11_e6_i80_o112_se0.25',
'r4_k5_s22_e6_i112_o192_se0.25',
'r1_k3_s11_e6_i192_o320_se0.25',
]
blocks_args = BlockDecoder.decode(blocks_args)
 global_params = GlobalParams(
 width_coefficient=width_coefficient,
depth_coefficient=depth_coefficient,
image_size=image_size,
dropout_rate=dropout_rate,
num_classes=num_classes,
batch_norm_momentum=0.99,
batch_norm_epsilon=1e-3,
drop_connect_rate=drop_connect_rate,
depth_divisor=8,
min_depth=None, 
)
 return blocks_args, global_params
GlobalParams = collections.namedtuple('GlobalParams', [
'width_coefficient', 'depth_coefficient', 'image_size', 'dropout_rate',
'num_classes', 'batch_norm_momentum', 'batch_norm_epsilon',
'drop_connect_rate', 'depth_divisor', 'min_depth'])
 BlockArgs = collections.namedtuple('BlockArgs', [
'num_repeat', 'kernel_size', 'stride', 'expand_ratio',
'input_filters', 'output_filters', 'se_ratio', 'id_skip'])
 GlobalParams.__new__.__defaults__ =(None,)* len(GlobalParams._fields)
 BlockArgs.__new__.__defaults__ =(None,)* len(BlockArgs._fields )
target_cols = target.drop('sig_id', axis=1 ).columns.values.tolist()<split>
VALID_MODELS =(
 'efficientnet-b0', 'efficientnet-b1', 'efficientnet-b2', 'efficientnet-b3',
'efficientnet-b4', 'efficientnet-b5', 'efficientnet-b6', 'efficientnet-b7',
'efficientnet-b8',
'efficientnet-l2' 
)
 class MBConvBlock(nn.Module):

 def __init__(self, block_args, global_params, image_size=None):
super().__init__()
self._block_args = block_args
self._bn_mom = 1 - global_params.batch_norm_momentum
self._bn_eps = global_params.batch_norm_epsilon
self.has_se =(self._block_args.se_ratio is not None)and(0 < self._block_args.se_ratio <= 1)
 self.id_skip = block_args.id_skip
inp = self._block_args.input_filters
oup = self._block_args.input_filters * self._block_args.expand_ratio
if self._block_args.expand_ratio != 1:
Conv2d = get_same_padding_conv2d(image_size=image_size)
 self._expand_conv = Conv2d(in_channels=inp, out_channels=oup, kernel_size=1, bias=False)
 self._bn0 = nn.BatchNorm2d(num_features=oup, momentum=self._bn_mom, eps=self._bn_eps)
 k = self._block_args.kernel_size
s = self._block_args.stride
Conv2d = get_same_padding_conv2d(image_size=image_size)
 self._depthwise_conv = Conv2d(
 in_channels=oup, out_channels=oup, groups=oup,
kernel_size=k, stride=s, bias=False)
 self._bn1 = nn.BatchNorm2d(num_features=oup, momentum=self._bn_mom, eps=self._bn_eps)
 image_size = calculate_output_image_size(image_size, s)
 if self.has_se:
Conv2d = get_same_padding_conv2d(image_size=(1,1))
num_squeezed_channels = max(1, int(self._block_args.input_filters * self._block_args.se_ratio))
self._se_reduce = Conv2d(in_channels=oup, out_channels=num_squeezed_channels, kernel_size=1)
 self._se_expand = Conv2d(in_channels=num_squeezed_channels, out_channels=oup, kernel_size=1)
 final_oup = self._block_args.output_filters
Conv2d = get_same_padding_conv2d(image_size=image_size)
 self._project_conv = Conv2d(in_channels=oup, out_channels=final_oup, kernel_size=1, bias=False)
 self._bn2 = nn.BatchNorm2d(num_features=final_oup, momentum=self._bn_mom, eps=self._bn_eps)
 self._swish = MemoryEfficientSwish()
def forward(self, inputs, drop_connect_rate=None):

 x = inputs
if self._block_args.expand_ratio != 1:
x = self._expand_conv(inputs)
 x = self._bn0(x)
 x = self._swish(x)
 x = self._depthwise_conv(x)
 x = self._bn1(x)
 x = self._swish(x)
 if self.has_se:
x_squeezed = F.adaptive_avg_pool2d(x, 1)
 x_squeezed = self._se_reduce(x_squeezed)
 x_squeezed = self._swish(x_squeezed)
 x_squeezed = self._se_expand(x_squeezed)
 x = torch.sigmoid(x_squeezed)* x
x = self._project_conv(x)
 x = self._bn2(x)
 input_filters, output_filters = self._block_args.input_filters, self._block_args.output_filters
if self.id_skip and self._block_args.stride == 1 and input_filters == output_filters:
if drop_connect_rate:
x = drop_connect(x, p=drop_connect_rate, training=self.training)
 x = x + inputs
return x
def set_swish(self, memory_efficient=True):

 self._swish = MemoryEfficientSwish() if memory_efficient else Swish()
class EfficientNet(nn.Module):

 def __init__(self, blocks_args=None, global_params=None):
super().__init__()
assert isinstance(blocks_args, list), 'blocks_args should be a list'
assert len(blocks_args)> 0, 'block args must be greater than 0'
self._global_params = global_params
self._blocks_args = blocks_args
bn_mom = 1 - self._global_params.batch_norm_momentum
bn_eps = self._global_params.batch_norm_epsilon
image_size = global_params.image_size
Conv2d = get_same_padding_conv2d(image_size=image_size)
 in_channels = 3
out_channels = round_filters(32, self._global_params)
 self._conv_stem = Conv2d(in_channels, out_channels, kernel_size=3, stride=2, bias=False)
 self._bn0 = nn.BatchNorm2d(num_features=out_channels, momentum=bn_mom, eps=bn_eps)
 image_size = calculate_output_image_size(image_size, 2)
 self._blocks = nn.ModuleList([])
 for block_args in self._blocks_args:
block_args = block_args._replace(
 input_filters=round_filters(block_args.input_filters, self._global_params),
output_filters=round_filters(block_args.output_filters, self._global_params),
num_repeat=round_repeats(block_args.num_repeat, self._global_params)
)
 self._blocks.append(MBConvBlock(block_args, self._global_params, image_size=image_size))
image_size = calculate_output_image_size(image_size, block_args.stride)
 if block_args.num_repeat > 1:
block_args = block_args._replace(input_filters=block_args.output_filters, stride=1)
 for _ in range(block_args.num_repeat - 1):
self._blocks.append(MBConvBlock(block_args, self._global_params, image_size=image_size))
in_channels = block_args.output_filters
out_channels = round_filters(1280, self._global_params)
 Conv2d = get_same_padding_conv2d(image_size=image_size)
 self._conv_head = Conv2d(in_channels, out_channels, kernel_size=1, bias=False)
 self._bn1 = nn.BatchNorm2d(num_features=out_channels, momentum=bn_mom, eps=bn_eps)
 self._avg_pooling = nn.AdaptiveAvgPool2d(1)
 self._dropout = nn.Dropout(self._global_params.dropout_rate)
 self._fc = nn.Linear(out_channels, self._global_params.num_classes)
 self._swish = MemoryEfficientSwish()
def set_swish(self, memory_efficient=True):

 self._swish = MemoryEfficientSwish() if memory_efficient else Swish()
for block in self._blocks:
block.set_swish(memory_efficient)
 def extract_endpoints(self, inputs):

 endpoints = dict()
x = self._swish(self._bn0(self._conv_stem(inputs)))
 prev_x = x
for idx, block in enumerate(self._blocks):
drop_connect_rate = self._global_params.drop_connect_rate
if drop_connect_rate:
drop_connect_rate *= float(idx)/ len(self._blocks)
 x = block(x, drop_connect_rate=drop_connect_rate)
 if prev_x.size(2)> x.size(2):
endpoints[f'reduction_{len(endpoints)+1}'] = prev_x
prev_x = x
x = self._swish(self._bn1(self._conv_head(x)))
 endpoints[f'reduction_{len(endpoints)+1}'] = x
return endpoints
def extract_features(self, inputs):

 x = self._swish(self._bn0(self._conv_stem(inputs)))
 for idx, block in enumerate(self._blocks):
drop_connect_rate = self._global_params.drop_connect_rate
if drop_connect_rate:
drop_connect_rate *= float(idx)/ len(self._blocks)
 x = block(x, drop_connect_rate=drop_connect_rate)
 x = self._swish(self._bn1(self._conv_head(x)))
 return x
def forward(self, inputs):

 x = self.extract_features(inputs)
 x = self._avg_pooling(x)
 x = x.flatten(start_dim=1)
 x = self._dropout(x)
 x = self._fc(x)
 return x
@classmethod
def from_name(cls, model_name, in_channels=3, **override_params):

 cls._check_model_name_is_valid(model_name)
 blocks_args, global_params = get_model_params(model_name, override_params)
 model = cls(blocks_args, global_params)
 model._change_in_channels(in_channels)
 return model
@classmethod
def from_pretrained(cls, model_name, weights_path=None, advprop=False,
in_channels=3, num_classes=1000, **override_params):

 model = cls.from_name(model_name, num_classes = num_classes, **override_params)
 load_pretrained_weights(model, model_name, weights_path=weights_path, load_fc=(num_classes == 1000), advprop=advprop)
 model._change_in_channels(in_channels)
 return model
@classmethod
def get_image_size(cls, model_name):

 cls._check_model_name_is_valid(model_name)
 _, _, res, _ = efficientnet_params(model_name)
 return res
@classmethod
def _check_model_name_is_valid(cls, model_name):

 if model_name not in VALID_MODELS:
raise ValueError('model_name should be one of: ' + ', '.join(VALID_MODELS))
def _change_in_channels(self, in_channels):

 if in_channels != 3:
Conv2d = get_same_padding_conv2d(image_size = self._global_params.image_size)
 out_channels = round_filters(32, self._global_params)
 self._conv_stem = Conv2d(in_channels, out_channels, kernel_size=3, stride=2, bias=False )
folds = train.copy()
mskf = MultilabelStratifiedKFold(n_splits=7)
 for f,(t_idx, v_idx)in enumerate(mskf.split(X=train, y=target)) :
folds.loc[v_idx, 'kfold'] = int(f)
 folds['kfold'] = folds['kfold'].astype(int)
 folds 
<categorify>
class ResNestSED(nn.Module):
def __init__(self, num_classes=264):
super().__init__()
self.interpolate_ratio = 30
base_model = ResNest(
 Bottleneck, [3, 4, 6, 3],
radix=1, groups=1, bottleneck_width=64,
deep_stem=True, stem_width=32, avg_down=True,
avd=True, avd_first=True)
 layers = list(base_model.children())[:-2]
self.encoder = nn.Sequential(*layers)
 in_features = base_model.fc.in_features
self.fc1 = nn.Linear(in_features, in_features, bias=True)
 self.att_block = AttBlock(in_features, num_classes, activation="sigmoid")
 self.init_weight()
def init_weight(self):
init_layer(self.fc1)
 def forward(self, input):
frames_num = input.size(3)
 x = self.encoder(input)
 x = torch.mean(x, dim=2)
 x1 = F.max_pool1d(x, kernel_size=3, stride=1, padding=1)
 x2 = F.avg_pool1d(x, kernel_size=3, stride=1, padding=1)
 x = x1 + x2
x = F.dropout(x, p=0.5, training=self.training)
 x = x.transpose(1, 2)
 x = F.relu_(self.fc1(x))
x = x.transpose(1, 2)
 x = F.dropout(x, p=0.5, training=self.training)
(clipwise_output, norm_att, segmentwise_output)= self.att_block(x)
 logit = torch.sum(norm_att * self.att_block.cla(x), dim=2)
 segmentwise_output = segmentwise_output.transpose(1, 2)
 framewise_output = interpolate(segmentwise_output,
self.interpolate_ratio)
 framewise_output = pad_framewise_output(framewise_output, frames_num)
 output_dict = {
"framewise_output": framewise_output,
"logit": logit,
"clipwise_output": clipwise_output
}
return output_dict
class MoADataset:
def __init__(self, features, targets):
self.features = features
self.targets = targets
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float),
'y' : torch.tensor(self.targets[idx, :], dtype=torch.float)
 }
return dct
class TestDataset:
def __init__(self, features):
self.features = features
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float)
 }
return dct<train_model>
class EfficientNetSED(nn.Module):
def __init__(self, base_model_name: str, pretrained=False,
num_classes=264):
super().__init__()
self.interpolate_ratio = 32
if pretrained:
self.base_model = EfficientNet.from_pretrained(base_model_name)
 else:
self.base_model = EfficientNet.from_name(base_model_name)
 in_features = self.base_model._fc.in_features
self.fc1 = nn.Linear(in_features, in_features, bias=True)
 self.att_block = AttBlock(in_features, num_classes, activation="sigmoid")
 self.init_weight()
def init_weight(self):
init_layer(self.fc1)
 def forward(self, input):
frames_num = input.size(3)
 x = self.base_model.extract_features(input)
 x = torch.mean(x, dim=2)
 x1 = F.max_pool1d(x, kernel_size=3, stride=1, padding=1)
 x2 = F.avg_pool1d(x, kernel_size=3, stride=1, padding=1)
 x = x1 + x2
x = F.dropout(x, p=0.5, training=self.training)
 x = x.transpose(1, 2)
 x = F.relu_(self.fc1(x))
x = x.transpose(1, 2)
 x = F.dropout(x, p=0.5, training=self.training)
(clipwise_output, _, segmentwise_output)= self.att_block(x)
 segmentwise_output = segmentwise_output.transpose(1, 2)
 framewise_output = interpolate(segmentwise_output,
self.interpolate_ratio)
 framewise_output = pad_framewise_output(framewise_output, frames_num)
 output_dict = {
"framewise_output": framewise_output,
"segmentwise_output": segmentwise_output,
"clipwise_output": clipwise_output
}
return output_dict
def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):
model.train()
final_loss = 0
for data in dataloader:
optimizer.zero_grad()
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
scheduler.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 return final_loss
def valid_fn(model, loss_fn, dataloader, device):
model.eval()
final_loss = 0
valid_preds = []
for data in dataloader:
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 return final_loss, valid_preds
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
inputs = data['x'].to(device)
 with torch.no_grad() :
outputs = model(inputs)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds 
<normalization>
resnest_model_config = {
"num_classes": 264
}
effnet_model_config = {
"num_classes": 264,
"base_model_name": "efficientnet-b0",
"pretrained": False
}
weights_path = {
"resnest": {
"base": ".. /input/birdcall-resnest-ema-all/ema.pth",
"th04": ".. /input/birdcall-resnest-ema-all-th04/ema.pth",
"th06": ".. /input/birdcall-resnest-ema-all-th06/ema.pth",
"ref2": ".. /input/birdcall-resnest-ema-all-refine2/ema.pth",
"ref2_th03": ".. /input/birdcall-resnest-ema-all-ref2-th03/ema.pth",
"ref2_th04": ".. /input/birdcall-resnest-ema-all-ref2-th04/ema.pth",
"ref2_th06": ".. /input/birdcall-resnest-ema-all-ref2-th06/ema.pth",
"ref2_th07": ".. /input/birdcall-resnest-ema-all-ref2-th07/ema.pth",
"ext": ".. /input/birdcall-resnest-emta-all-ext-ref2-th04/ema.pth"
},
"effnet": {
"eff_th04": ".. /input/birdcall-effnet-ema-all-focal-ref2-th04/ema.pth",
"eff_th05": ".. /input/birdcall-effnet-b0-ema-all-ref2/ema.pth"
}
}
ratio = {
"base": 0.05,
"th06": 0.05,
"th04": 0.01,
"ref2": 0.05,
"ref2_th03": 0.25,
"ref2_th04": 0.14,
"ref2_th06": 0.05,
"ref2_th07": 0.01,
"eff_th04": 0.13,
"eff_th05": 0.01,
"ext": 0.25
}
melspectrogram_parameters = {
"n_mels": 128,
"fmin": 20,
"fmax": 16000
}
pcen_parameters = {
"gain": 0.98,
"bias": 2,
"power": 0.5,
"time_constant": 0.4,
"eps": 0.000001
}
class SmoothBCEwLogits(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=0.0):
super().__init__(weight=weight, reduction=reduction)
 self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
@staticmethod
def _smooth(targets:torch.Tensor, n_labels:int, smoothing=0.0):
assert 0 <= smoothing < 1
with torch.no_grad() :
targets = targets *(1.0 - smoothing)+ 0.5 * smoothing
return targets
def forward(self, inputs, targets):
targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1),
self.smoothing)
 loss = F.binary_cross_entropy_with_logits(inputs, targets,self.weight)
 if self.reduction == 'sum':
loss = loss.sum()
elif self.reduction == 'mean':
loss = loss.mean()
return loss 
<choose_model_class>
BIRD_CODE = {
'aldfly': 0, 'ameavo': 1, 'amebit': 2, 'amecro': 3, 'amegfi': 4,
'amekes': 5, 'amepip': 6, 'amered': 7, 'amerob': 8, 'amewig': 9,
'amewoo': 10, 'amtspa': 11, 'annhum': 12, 'astfly': 13, 'baisan': 14,
'baleag': 15, 'balori': 16, 'banswa': 17, 'barswa': 18, 'bawwar': 19,
'belkin1': 20, 'belspa2': 21, 'bewwre': 22, 'bkbcuc': 23, 'bkbmag1': 24,
'bkbwar': 25, 'bkcchi': 26, 'bkchum': 27, 'bkhgro': 28, 'bkpwar': 29,
'bktspa': 30, 'blkpho': 31, 'blugrb1': 32, 'blujay': 33, 'bnhcow': 34,
'boboli': 35, 'bongul': 36, 'brdowl': 37, 'brebla': 38, 'brespa': 39,
'brncre': 40, 'brnthr': 41, 'brthum': 42, 'brwhaw': 43, 'btbwar': 44,
'btnwar': 45, 'btywar': 46, 'buffle': 47, 'buggna': 48, 'buhvir': 49,
'bulori': 50, 'bushti': 51, 'buwtea': 52, 'buwwar': 53, 'cacwre': 54,
'calgul': 55, 'calqua': 56, 'camwar': 57, 'cangoo': 58, 'canwar': 59,
'canwre': 60, 'carwre': 61, 'casfin': 62, 'caster1': 63, 'casvir': 64,
'cedwax': 65, 'chispa': 66, 'chiswi': 67, 'chswar': 68, 'chukar': 69,
'clanut': 70, 'cliswa': 71, 'comgol': 72, 'comgra': 73, 'comloo': 74,
'commer': 75, 'comnig': 76, 'comrav': 77, 'comred': 78, 'comter': 79,
'comyel': 80, 'coohaw': 81, 'coshum': 82, 'cowscj1': 83, 'daejun': 84,
'doccor': 85, 'dowwoo': 86, 'dusfly': 87, 'eargre': 88, 'easblu': 89,
'easkin': 90, 'easmea': 91, 'easpho': 92, 'eastow': 93, 'eawpew': 94,
'eucdov': 95, 'eursta': 96, 'evegro': 97, 'fiespa': 98, 'fiscro': 99,
'foxspa': 100, 'gadwal': 101, 'gcrfin': 102, 'gnttow': 103, 'gnwtea': 104,
'gockin': 105, 'gocspa': 106, 'goleag': 107, 'grbher3': 108, 'grcfly': 109,
'greegr': 110, 'greroa': 111, 'greyel': 112, 'grhowl': 113, 'grnher': 114,
'grtgra': 115, 'grycat': 116, 'gryfly': 117, 'haiwoo': 118, 'hamfly': 119,
'hergul': 120, 'herthr': 121, 'hoomer': 122, 'hoowar': 123, 'horgre': 124,
'horlar': 125, 'houfin': 126, 'houspa': 127, 'houwre': 128, 'indbun': 129,
'juntit1': 130, 'killde': 131, 'labwoo': 132, 'larspa': 133, 'lazbun': 134,
'leabit': 135, 'leafly': 136, 'leasan': 137, 'lecthr': 138, 'lesgol': 139,
'lesnig': 140, 'lesyel': 141, 'lewwoo': 142, 'linspa': 143, 'lobcur': 144,
'lobdow': 145, 'logshr': 146, 'lotduc': 147, 'louwat': 148, 'macwar': 149,
'magwar': 150, 'mallar3': 151, 'marwre': 152, 'merlin': 153, 'moublu': 154,
'mouchi': 155, 'moudov': 156, 'norcar': 157, 'norfli': 158, 'norhar2': 159,
'normoc': 160, 'norpar': 161, 'norpin': 162, 'norsho': 163, 'norwat': 164,
'nrwswa': 165, 'nutwoo': 166, 'olsfly': 167, 'orcwar': 168, 'osprey': 169,
'ovenbi1': 170, 'palwar': 171, 'pasfly': 172, 'pecsan': 173, 'perfal': 174,
'phaino': 175, 'pibgre': 176, 'pilwoo': 177, 'pingro': 178, 'pinjay': 179,
'pinsis': 180, 'pinwar': 181, 'plsvir': 182, 'prawar': 183, 'purfin': 184,
'pygnut': 185, 'rebmer': 186, 'rebnut': 187, 'rebsap': 188, 'rebwoo': 189,
'redcro': 190, 'redhea': 191, 'reevir1': 192, 'renpha': 193, 'reshaw': 194,
'rethaw': 195, 'rewbla': 196, 'ribgul': 197, 'rinduc': 198, 'robgro': 199,
'rocpig': 200, 'rocwre': 201, 'rthhum': 202, 'ruckin': 203, 'rudduc': 204,
'rufgro': 205, 'rufhum': 206, 'rusbla': 207, 'sagspa1': 208, 'sagthr': 209,
'savspa': 210, 'saypho': 211, 'scatan': 212, 'scoori': 213, 'semplo': 214,
'semsan': 215, 'sheowl': 216, 'shshaw': 217, 'snobun': 218, 'snogoo': 219,
'solsan': 220, 'sonspa': 221, 'sora': 222, 'sposan': 223, 'spotow': 224,
'stejay': 225, 'swahaw': 226, 'swaspa': 227, 'swathr': 228, 'treswa': 229,
'truswa': 230, 'tuftit': 231, 'tunswa': 232, 'veery': 233, 'vesspa': 234,
'vigswa': 235, 'warvir': 236, 'wesblu': 237, 'wesgre': 238, 'weskin': 239,
'wesmea': 240, 'wessan': 241, 'westan': 242, 'wewpew': 243, 'whbnut': 244,
'whcspa': 245, 'whfibi': 246, 'whtspa': 247, 'whtswi': 248, 'wilfly': 249,
'wilsni1': 250, 'wiltur': 251, 'winwre3': 252, 'wlswar': 253, 'wooduc': 254,
'wooscj2': 255, 'woothr': 256, 'y00475': 257, 'yebfly': 258, 'yebsap': 259,
'yehbla': 260, 'yelwar': 261, 'yerwar': 262, 'yetvir': 263
}
INV_BIRD_CODE = {v: k for k, v in BIRD_CODE.items() }
class Model(nn.Module):
def __init__(self, num_features, num_targets, hidden_size):
super(Model, self ).__init__()
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dense1 = nn.Linear(num_features, hidden_size)
 self.batch_norm2 = nn.BatchNorm1d(hidden_size)
 self.dropout2 = nn.Dropout(0.2619422201258426)
 self.dense2 = nn.Linear(hidden_size, hidden_size)
 self.batch_norm3 = nn.BatchNorm1d(hidden_size)
 self.dropout3 = nn.Dropout(0.2619422201258426)
 self.dense3 = nn.Linear(hidden_size, num_targets)
 def forward(self, x):
x = self.batch_norm1(x)
 x = F.leaky_relu(self.dense1(x))
x = self.batch_norm2(x)
 x = self.dropout2(x)
 x = F.leaky_relu(self.dense2(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = self.dense3(x)
 return x
class LabelSmoothingLoss(nn.Module):
def __init__(self, classes, smoothing=0.0, dim=-1):
super(LabelSmoothingLoss, self ).__init__()
self.confidence = 1.0 - smoothing
self.smoothing = smoothing
self.cls = classes
self.dim = dim
def forward(self, pred, target):
pred = pred.log_softmax(dim=self.dim)
 with torch.no_grad() :
true_dist = torch.zeros_like(pred)
 true_dist.fill_(self.smoothing /(self.cls - 1))
true_dist.scatter_(1, target.data.unsqueeze(1), self.confidence)
 return torch.mean(torch.sum(-true_dist * pred, dim=self.dim))<categorify>
thresholds = {
'aldfly': 0.8300000000000001, 'ameavo': 0.35000000000000003, 'amebit': 0.73, 'amecro': 0.35000000000000003, 'amegfi': 0.53,
'amekes': 0.47000000000000003, 'amepip': 0.78, 'amered': 0.87, 'amerob': 0.26, 'amewig': 0.98,
'amewoo': 0.81, 'amtspa': 0.74, 'annhum': 0.78, 'astfly': 0.9500000000000001, 'baisan': 0.9400000000000001,
'baleag': 0.61, 'balori': 0.74, 'banswa': 0.85, 'barswa': 0.6, 'bawwar': 0.4,
'belkin1': 0.41000000000000003, 'belspa2': 0.79, 'bewwre': 0.9, 'bkbcuc': 0.76, 'bkbmag1': 0.67,
'bkbwar': 0.54, 'bkcchi': 0.58, 'bkchum': 0.68, 'bkhgro': 0.52, 'bkpwar': 0.5700000000000001,
'bktspa': 0.87, 'blkpho': 0.7000000000000001, 'blugrb1': 0.4, 'blujay': 0.66, 'bnhcow': 0.56,
'boboli': 0.6, 'bongul': 0.7000000000000001, 'brdowl': 0.98, 'brebla': 0.77, 'brespa': 0.9,
'brncre': 0.6900000000000001, 'brnthr': 0.97, 'brthum': 0.86, 'brwhaw': 0.9500000000000001, 'btbwar': 0.89,
'btnwar': 0.9, 'btywar': 0.98, 'buffle': 0.9400000000000001, 'buggna': 0.67, 'buhvir': 0.34,
'bulori': 0.68, 'bushti': 0.67, 'buwtea': 0.3, 'buwwar': 0.61, 'cacwre': 0.59,
'calgul': 0.89, 'calqua': 0.86, 'camwar': 0.67, 'cangoo': 0.46, 'canwar': 0.86,
'canwre': 0.76, 'carwre': 0.53, 'casfin': 0.6900000000000001, 'caster1': 0.48, 'casvir': 0.79,
'cedwax': 0.5, 'chispa': 0.43, 'chiswi': 0.6, 'chswar': 0.9500000000000001, 'chukar': 0.47000000000000003,
'clanut': 0.99, 'cliswa': 0.51, 'comgol': 0.5, 'comgra': 0.63, 'comloo': 0.29,
'commer': 0.79, 'comnig': 0.81, 'comrav': 0.42, 'comred': 0.76, 'comter': 0.67,
'comyel': 0.73, 'coohaw': 0.46, 'coshum': 0.64, 'cowscj1': 0.56, 'daejun': 0.65,
'doccor': 0.72, 'dowwoo': 0.68, 'dusfly': 0.8300000000000001, 'eargre': 0.9, 'easblu': 0.71,
'easkin': 0.97, 'easmea': 0.81, 'easpho': 0.96, 'eastow': 0.5, 'eawpew': 0.6900000000000001,
'eucdov': 0.65, 'eursta': 0.47000000000000003, 'evegro': 0.81, 'fiespa': 0.63, 'fiscro': 0.91,
'foxspa': 0.55, 'gadwal': 0.77, 'gcrfin': 0.6900000000000001, 'gnttow': 0.6900000000000001, 'gnwtea': 0.67,
'gockin': 0.66, 'gocspa': 0.86, 'goleag': 0.48, 'grbher3': 0.59, 'grcfly': 0.91,
'greegr': 0.6900000000000001, 'greroa': 0.81, 'greyel': 0.91, 'grhowl': 0.8200000000000001, 'grnher': 0.59,
'grtgra': 0.52, 'grycat': 0.65, 'gryfly': 0.5, 'haiwoo': 0.76, 'hamfly': 0.51,
'hergul': 0.92, 'herthr': 0.88, 'hoomer': 0.64, 'hoowar': 0.85, 'horgre': 0.47000000000000003,
'horlar': 0.78, 'houfin': 0.36, 'houspa': 0.44, 'houwre': 0.38, 'indbun': 0.71,
'juntit1': 0.96, 'killde': 0.6900000000000001, 'labwoo': 0.64, 'larspa': 0.67, 'lazbun': 0.6900000000000001,
'leabit': 0.88, 'leafly': 0.35000000000000003, 'leasan': 0.62, 'lecthr': 0.9400000000000001, 'lesgol': 0.72,
'lesnig': 0.6, 'lesyel': 0.37, 'lewwoo': 0.62, 'linspa': 0.88, 'lobcur': 0.73,
'lobdow': 0.49, 'logshr': 0.93, 'lotduc': 0.43, 'louwat': 0.6900000000000001, 'macwar': 0.76,
'magwar': 0.99, 'mallar3': 0.68, 'marwre': 0.81, 'merlin': 0.77, 'moublu': 0.79,
'mouchi': 0.85, 'moudov': 0.59, 'norcar': 0.38, 'norfli': 0.5, 'norhar2': 0.48,
'normoc': 0.42, 'norpar': 0.51, 'norpin': 0.84, 'norsho': 0.48, 'norwat': 0.56,
'nrwswa': 0.36, 'nutwoo': 0.38, 'olsfly': 0.96, 'orcwar': 0.78, 'osprey': 0.97,
'ovenbi1': 0.56, 'palwar': 0.48, 'pasfly': 0.9, 'pecsan': 0.41000000000000003, 'perfal': 0.6900000000000001,
'phaino': 0.91, 'pibgre': 0.48, 'pilwoo': 0.64, 'pingro': 0.84, 'pinjay': 0.92,
'pinsis': 0.39, 'pinwar': 0.88, 'plsvir': 0.6, 'prawar': 0.9, 'purfin': 0.55,
'pygnut': 0.9400000000000001, 'rebmer': 0.76, 'rebnut': 0.33, 'rebsap': 0.42, 'rebwoo': 0.38,
'redcro': 0.77, 'redhea': 0.86, 'reevir1': 0.72, 'renpha': 0.48, 'reshaw': 0.5700000000000001,
'rethaw': 0.79, 'rewbla': 0.43, 'ribgul': 0.3, 'rinduc': 0.47000000000000003, 'robgro': 0.92,
'rocpig': 0.71, 'rocwre': 0.58, 'rthhum': 0.44, 'ruckin': 0.85, 'rudduc': 0.67,
'rufgro': 0.27, 'rufhum': 0.86, 'rusbla': 0.8200000000000001, 'sagspa1': 0.75, 'sagthr': 0.86,
'savspa': 0.81, 'saypho': 0.85, 'scatan': 0.96, 'scoori': 0.93, 'semplo': 0.46,
'semsan': 0.96, 'sheowl': 0.43, 'shshaw': 0.44, 'snobun': 0.6, 'snogoo': 0.97,
'solsan': 0.73, 'sonspa': 0.59, 'sora': 0.72, 'sposan': 0.5700000000000001, 'spotow': 0.55,
'stejay': 0.6900000000000001, 'swahaw': 0.58, 'swaspa': 0.6, 'swathr': 0.67, 'treswa': 0.98,
'truswa': 0.93, 'tuftit': 0.71, 'tunswa': 0.55, 'veery': 0.91, 'vesspa': 0.96,
'vigswa': 0.45, 'warvir': 0.86, 'wesblu': 0.65, 'wesgre': 0.6, 'weskin': 0.8200000000000001,
'wesmea': 0.58, 'wessan': 0.63, 'westan': 0.47000000000000003, 'wewpew': 0.78, 'whbnut': 0.48,
'whcspa': 0.72, 'whfibi': 0.66, 'whtspa': 0.76, 'whtswi': 0.96, 'wilfly': 0.68,
'wilsni1': 0.36, 'wiltur': 0.41000000000000003, 'winwre3': 0.68, 'wlswar': 0.77, 'wooduc': 0.61,
'wooscj2': 0.93, 'woothr': 0.8200000000000001, 'y00475': 0.45, 'yebfly': 0.52, 'yebsap': 0.39,
'yehbla': 0.6900000000000001, 'yelwar': 0.66, 'yerwar': 0.64, 'yetvir': 0.9
}
for key in thresholds:
thresholds[key] = 0.4
def process_data(data):
data = pd.get_dummies(data, columns=['cp_time','cp_dose'])
 return data<define_variables>
def get_model(resnest_config: dict, effnet_config: dict, weights_path: dict):
models = {}
device = torch.device("cuda")
 for model_key in weights_path:
path_dict = weights_path[model_key]
for path_key in path_dict:
if model_key == "resnest":
model = ResNestSED(**resnest_config)
 else:
model = EfficientNetSED(**effnet_config)
 checkpoint = torch.load(path_dict[path_key])
 model_state_dict = {}
for key in checkpoint["model_state_dict"]:
if key == "n_averaged":
continue
new_key = key.replace("module.", "")
 model_state_dict[new_key] = checkpoint["model_state_dict"][key]
model.load_state_dict(model_state_dict)
 model.to(device)
 model.eval()
models[path_key] = model
return models
feature_cols = [c for c in process_data(folds ).columns if c not in target_cols]
feature_cols = [c for c in feature_cols if c not in ['kfold','sig_id']]
len(feature_cols )<define_variables>
def normalize_melspec(X: np.ndarray):
eps = 1e-6
mean = X.mean()
X = X - mean
std = X.std()
Xstd = X /(std + eps)
 norm_min, norm_max = Xstd.min() , Xstd.max()
if(norm_max - norm_min)> eps:
V = Xstd
V[V < norm_min] = norm_min
V[V > norm_max] = norm_max
V = 255 *(V - norm_min)/(norm_max - norm_min)
 V = V.astype(np.uint8)
 else:
V = np.zeros_like(Xstd, dtype=np.uint8)
 return V
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS = 26
BATCH_SIZE = 256
LEARNING_RATE = 6e-4
WEIGHT_DECAY = 1e-5
NFOLDS = 7
EARLY_STOPPING_STEPS = 10
EARLY_STOP = True
num_features=len(feature_cols)
 num_targets=len(target_cols)
 hidden_size=2028<load_from_csv>
def prediction_for_clip(test_df: pd.DataFrame,
clip: np.ndarray,
models,
melspectrogram_parameters={},
pcen_parameters={}):
PERIOD = 30
images = []
y = clip.astype(np.float32)
 len_y = len(y)
 start = 0
end = PERIOD * TARGET_SR
while True:
y_batch = y[start:end].astype(np.float32)
 if len(y_batch)> 0:
max_vol = np.abs(y_batch ).max()
if max_vol > 0:
y_batch = np.asfortranarray(y_batch * 1 / max_vol)
 if len(y_batch)!= PERIOD * TARGET_SR:
y_pad = np.zeros(PERIOD * TARGET_SR, dtype=np.float32)
 y_pad[:len(y_batch)] = y_batch
melspec = librosa.feature.melspectrogram(y_pad,
sr=TARGET_SR,
**melspectrogram_parameters)
 pcen = librosa.pcen(melspec, sr=TARGET_SR, **pcen_parameters)
 clean_mel = librosa.power_to_db(melspec ** 1.5)
 melspec = librosa.power_to_db(melspec ).astype(np.float32)
 norm_melspec = normalize_melspec(melspec)
 norm_pcen = normalize_melspec(pcen)
 norm_clean_mel = normalize_melspec(clean_mel)
 image = np.stack([norm_melspec, norm_pcen, norm_clean_mel], axis=-1)
 height, width, _ = image.shape
image = cv2.resize(image,(int(width * 224 / height), 224))
image = np.moveaxis(image, 2, 0)
 image =(image / 255.0 ).astype(np.float32)
 images.append(image)
 break
start = end
end += PERIOD * TARGET_SR
melspec = librosa.feature.melspectrogram(y_batch,
sr=TARGET_SR,
**melspectrogram_parameters)
 pcen = librosa.pcen(melspec, sr=TARGET_SR, **pcen_parameters)
 clean_mel = librosa.power_to_db(melspec ** 1.5)
 melspec = librosa.power_to_db(melspec ).astype(np.float32)
 norm_melspec = normalize_melspec(melspec)
 norm_pcen = normalize_melspec(pcen)
 norm_clean_mel = normalize_melspec(clean_mel)
 image = np.stack([norm_melspec, norm_pcen, norm_clean_mel], axis=-1)
 height, width, _ = image.shape
image = cv2.resize(image,(int(width * 224 / height), 224))
image = np.moveaxis(image, 2, 0)
 image =(image / 255.0 ).astype(np.float32)
 images.append(image)
 array = np.asarray(images)
 tensors = torch.from_numpy(array)
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 estimated_event_list = []
global_time = 0.0
site = test_df["site"].values[0]
audio_id = test_df["audio_id"].values[0]
for image in progress_bar(tensors):
image = image.view(1, image.size(0), image.size(1), image.size(2))
image = image.to(device)
 outputs = {}
with torch.no_grad() :
for key in models:
prediction = models[key](image)
 framewise_outputs = prediction["framewise_output"].detach(
 ).cpu().numpy() [0]
outputs[key] = framewise_outputs
key = list(outputs.keys())[0]
framewise_outputs = np.zeros_like(outputs[key], dtype=np.float32)
 for key in outputs:
framewise_outputs += ratio[key] * outputs[key]
thresholded = np.zeros_like(framewise_outputs)
 for i in range(len(thresholds)) :
thresholded[:, i] = framewise_outputs[:, i] >= thresholds[INV_BIRD_CODE[i]]
sec_per_frame = PERIOD / thresholded.shape[0]
for target_idx in range(thresholded.shape[1]):
if thresholded[:, target_idx].mean() == 0:
pass
else:
detected = np.argwhere(thresholded[:, target_idx] ).reshape(-1)
 head_idx = 0
tail_idx = 0
while True:
if(tail_idx + 1 == len(detected)) or(
 detected[tail_idx + 1] -
detected[tail_idx] != 1):
onset = sec_per_frame * detected[
head_idx] + global_time
offset = sec_per_frame * detected[
tail_idx] + global_time
onset_idx = detected[head_idx]
offset_idx = detected[tail_idx]
max_confidence = framewise_outputs[
onset_idx:offset_idx, target_idx].max()
mean_confidence = framewise_outputs[
onset_idx:offset_idx, target_idx].mean()
estimated_event = {
"site": site,
"audio_id": audio_id,
"ebird_code": INV_BIRD_CODE[target_idx],
"onset": onset,
"offset": offset,
"max_confidence": max_confidence,
"mean_confidence": mean_confidence
}
estimated_event_list.append(estimated_event)
 head_idx = tail_idx + 1
tail_idx = tail_idx + 1
if head_idx >= len(detected):
break
else:
tail_idx += 1
global_time += PERIOD
prediction_df = pd.DataFrame(estimated_event_list)
 return prediction_df
train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv' )<prepare_x_and_y>
def prediction(test_df: pd.DataFrame,
test_audio: Path,
resnest_config: dict,
effnet_config: dict,
weights_path: dict,
melspectrogram_parameters={},
pcen_parameters={}):
models = get_model(resnest_config, effnet_config, weights_path)
 unique_audio_id = test_df.audio_id.unique()
warnings.filterwarnings("ignore")
 prediction_dfs = []
for audio_id in unique_audio_id:
with timer(f"Loading {audio_id}"):
clip, _ = librosa.load(test_audio /(audio_id + ".mp3"),
sr=TARGET_SR,
mono=True,
res_type="kaiser_fast")
 test_df_for_audio_id = test_df.query(
 f"audio_id == '{audio_id}'" ).reset_index(drop=True)
 with timer(f"Prediction on {audio_id}"):
prediction_df = prediction_for_clip(test_df_for_audio_id,
clip=clip,
models=models,
melspectrogram_parameters=melspectrogram_parameters,
pcen_parameters=pcen_parameters)
 prediction_dfs.append(prediction_df)
 prediction_df = pd.concat(prediction_dfs, axis=0, sort=False ).reset_index(drop=True)
 return prediction_df
target = train[train_targets_scored.columns]
target_cols = target.drop('sig_id', axis=1 ).columns.values.tolist()<define_variables>
prediction_df = prediction(test_df=test,
test_audio=test_audio,
resnest_config=resnest_model_config,
effnet_config=effnet_model_config,
weights_path=weights_path,
melspectrogram_parameters=melspectrogram_parameters,
pcen_parameters=pcen_parameters )
feature_cols = [c for c in process_data(folds ).columns if c not in target_cols]
feature_cols = [c for c in feature_cols if c not in ['kfold','sig_id']]
len(feature_cols )<split>
labels = {}
for audio_id, sub_df in prediction_df.groupby("audio_id"):
events = sub_df[["ebird_code", "onset", "offset", "max_confidence", "site"]].values
n_events = len(events)
 site = events[0][4]
for i in range(n_events):
event = events[i][0]
onset = events[i][1]
offset = events[i][2]
if site in {"site_1", "site_2"}:
start_section = int(( onset // 5)* 5)+ 5
end_section = int(( offset // 5)* 5)+ 5
cur_section = start_section
row_id = f"{site}_{audio_id}_{start_section}"
if labels.get(row_id)is not None:
labels[row_id].add(event)
 else:
labels[row_id] = set()
labels[row_id].add(event)
 while cur_section != end_section:
cur_section += 5
row_id = f"{site}_{audio_id}_{cur_section}"
if labels.get(row_id)is not None:
labels[row_id].add(event)
 else:
labels[row_id] = set()
labels[row_id].add(event)
 else:
row_id = f"{site}_{audio_id}"
if labels.get(row_id)is not None:
labels[row_id].add(event)
 else:
labels[row_id] = set()
labels[row_id].add(event)
folds = train.copy()
mskf = MultilabelStratifiedKFold(n_splits=7)
 for f,(t_idx, v_idx)in enumerate(mskf.split(X=train, y=target)) :
folds.loc[v_idx, 'kfold'] = int(f)
 folds['kfold'] = folds['kfold'].astype(int)
 folds<prepare_x_and_y>
for key in labels:
labels[key] = " ".join(sorted(list(labels[key])))
def run_training(fold, seed):
seed_everything(seed)
 train = process_data(folds)
 test_ = process_data(test)
 trn_idx = train[train['kfold'] != fold].index
val_idx = train[train['kfold'] == fold].index
train_df = train[train['kfold'] != fold].reset_index(drop=True)
 valid_df = train[train['kfold'] == fold].reset_index(drop=True)
 x_train, y_train = train_df[feature_cols].values, train_df[target_cols].values
x_valid, y_valid = valid_df[feature_cols].values, valid_df[target_cols].values
train_dataset = MoADataset(x_train, y_train)
 valid_dataset = MoADataset(x_valid, y_valid)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=402,
hidden_size=2028, 
)
 model.load_state_dict(torch.load(".. /input/nonscored-model-fold1pth/FOLD1_non_scored.pth"))
model.batch_norm3 = nn.BatchNorm1d(hidden_size)
 model.dropout3 = nn.Dropout(0.5)
 model.dense3 = nn.Linear(hidden_size,206)
 model.to(DEVICE)
 optimizer = torch.optim.Adam(model.parameters() , lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY)
 scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=0.1, div_factor=1e3,
max_lr=1e-2, epochs=EPOCHS, steps_per_epoch=len(trainloader))
loss_fn = nn.BCEWithLogitsLoss()
loss_tr = SmoothBCEwLogits(smoothing =0.001)
 early_stopping_steps = EARLY_STOPPING_STEPS
early_step = 0
oof = np.zeros(( len(train), target.iloc[:, 1:].shape[1]))
best_loss = np.inf
for epoch in range(EPOCHS):
train_loss = train_fn(model, optimizer,scheduler, loss_tr, trainloader, DEVICE)
 print(f"FOLD: {fold}, EPOCH: {epoch}, train_loss: {train_loss}")
 valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 print(f"FOLD: {fold}, EPOCH: {epoch}, valid_loss: {valid_loss}")
 if valid_loss < best_loss:
best_loss = valid_loss
oof[val_idx] = valid_preds
torch.save(model.state_dict() , f"FOLD{fold}_scored.pth")
 elif(EARLY_STOP == True):
early_step += 1
if(early_step >= early_stopping_steps):
break
x_test = test_[feature_cols].values
testdataset = TestDataset(x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.load_state_dict(torch.load(f"FOLD{fold}_scored.pth"))
model.to(DEVICE)
 predictions = np.zeros(( len(test_), target.iloc[:, 1:].shape[1]))
predictions = inference_fn(model, testloader, DEVICE)
 return oof, predictions<categorify>
row_ids = list(labels.keys())
 birds = list(labels.values())
 post_processed = pd.DataFrame({
"row_id": row_ids,
"birds": birds
})
 post_processed.head()
<categorify><EOS> 
all_row_id = test[["row_id"]]
submission = all_row_id.merge(post_processed, on="row_id", how="left")
 submission = submission.fillna("nocall")
 submission.to_csv("submission.csv", index=False)
 submission.head(20 )