Kaggle competition: Costa Rican Household Poverty Level Prediction
import os
import sys
import pickle as pkl
import librosa
import numpy as np
import pandas as pd
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.utils.data import DataLoader, Dataset
from tqdm import tqdm
from pathlib import Path
from scipy.special import logit, expit
from skimage.transform import rescale, resize, downscale_local_mean 
<import_modules>
%matplotlib inline
GlobalParams = collections.namedtuple('GlobalParams', [
'batch_norm_momentum', 'batch_norm_epsilon', 'dropout_rate',
'num_classes', 'width_coefficient', 'depth_coefficient',
'depth_divisor', 'min_depth', 'drop_connect_rate', 'image_size'])
 BlockArgs = collections.namedtuple('BlockArgs', [
'kernel_size', 'num_repeat', 'input_filters', 'output_filters',
'expand_ratio', 'id_skip', 'stride', 'se_ratio'])
 GlobalParams.__new__.__defaults__ =(None,)* len(GlobalParams._fields)
 BlockArgs.__new__.__defaults__ =(None,)* len(BlockArgs._fields)
 class SwishImplementation(torch.autograd.Function):
@staticmethod
def forward(ctx, i):
result = i * torch.sigmoid(i)
 ctx.save_for_backward(i)
 return result
@staticmethod
def backward(ctx, grad_output):
i = ctx.saved_variables[0]
sigmoid_i = torch.sigmoid(i)
 return grad_output *(sigmoid_i *(1 + i *(1 - sigmoid_i)))
 class MemoryEfficientSwish(nn.Module):
def forward(self, x):
return SwishImplementation.apply(x)
 class Swish(nn.Module):
def forward(self, x):
return x * torch.sigmoid(x)
 def round_filters(filters, global_params):

 multiplier = global_params.width_coefficient
if not multiplier:
return filters
divisor = global_params.depth_divisor
min_depth = global_params.min_depth
filters *= multiplier
min_depth = min_depth or divisor
new_filters = max(min_depth, int(filters + divisor / 2)// divisor * divisor)
 if new_filters < 0.9 * filters:
new_filters += divisor
return int(new_filters)
 def round_repeats(repeats, global_params):

 multiplier = global_params.depth_coefficient
if not multiplier:
return repeats
return int(math.ceil(multiplier * repeats))
def drop_connect(inputs, p, training):

 if not training: return inputs
batch_size = inputs.shape[0]
keep_prob = 1 - p
random_tensor = keep_prob
random_tensor += torch.rand([batch_size, 1, 1, 1], dtype=inputs.dtype, device=inputs.device)
 binary_tensor = torch.floor(random_tensor)
 output = inputs / keep_prob * binary_tensor
return output
def get_same_padding_conv2d(image_size=None):

 if image_size is None:
return Conv2dDynamicSamePadding
else:
return partial(Conv2dStaticSamePadding, image_size=image_size)
 class Conv2dDynamicSamePadding(nn.Conv2d):

 def __init__(self, in_channels, out_channels, kernel_size, stride=1, dilation=1, groups=1, bias=True):
super().__init__(in_channels, out_channels, kernel_size, stride, 0, dilation, groups, bias)
 self.stride = self.stride if len(self.stride)== 2 else [self.stride[0]] * 2
def forward(self, x):
ih, iw = x.size() [-2:]
kh, kw = self.weight.size() [-2:]
sh, sw = self.stride
oh, ow = math.ceil(ih / sh), math.ceil(iw / sw)
 pad_h = max(( oh - 1)* self.stride[0] +(kh - 1)* self.dilation[0] + 1 - ih, 0)
 pad_w = max(( ow - 1)* self.stride[1] +(kw - 1)* self.dilation[1] + 1 - iw, 0)
 if pad_h > 0 or pad_w > 0:
x = F.pad(x, [pad_w // 2, pad_w - pad_w // 2, pad_h // 2, pad_h - pad_h // 2])
 return F.conv2d(x, self.weight, self.bias, self.stride, self.padding, self.dilation, self.groups)
 class Conv2dStaticSamePadding(nn.Conv2d):

 def __init__(self, in_channels, out_channels, kernel_size, image_size=None, **kwargs):
super().__init__(in_channels, out_channels, kernel_size, **kwargs)
 self.stride = self.stride if len(self.stride)== 2 else [self.stride[0]] * 2
assert image_size is not None
ih, iw = image_size if type(image_size)== list else [image_size, image_size]
kh, kw = self.weight.size() [-2:]
sh, sw = self.stride
oh, ow = math.ceil(ih / sh), math.ceil(iw / sw)
 pad_h = max(( oh - 1)* self.stride[0] +(kh - 1)* self.dilation[0] + 1 - ih, 0)
 pad_w = max(( ow - 1)* self.stride[1] +(kw - 1)* self.dilation[1] + 1 - iw, 0)
 if pad_h > 0 or pad_w > 0:
self.static_padding = nn.ZeroPad2d(( pad_w // 2, pad_w - pad_w // 2, pad_h // 2, pad_h - pad_h // 2))
else:
self.static_padding = Identity()
def forward(self, x):
x = self.static_padding(x)
 x = F.conv2d(x, self.weight, self.bias, self.stride, self.padding, self.dilation, self.groups)
 return x
class Identity(nn.Module):
def __init__(self,):
super(Identity, self ).__init__()
def forward(self, input):
return input
def efficientnet_params(model_name):

 params_dict = {
'efficientnet-b0':(1.0, 1.0, 224, 0.2),
'efficientnet-b1':(1.0, 1.1, 240, 0.2),
'efficientnet-b2':(1.1, 1.2, 260, 0.3),
'efficientnet-b3':(1.2, 1.4, 300, 0.3),
'efficientnet-b4':(1.4, 1.8, 380, 0.4),
'efficientnet-b5':(1.6, 2.2, 456, 0.4),
'efficientnet-b6':(1.8, 2.6, 528, 0.5),
'efficientnet-b7':(2.0, 3.1, 600, 0.5),
'efficientnet-b8':(2.2, 3.6, 672, 0.5),
'efficientnet-l2':(4.3, 5.3, 800, 0.5),
}
return params_dict[model_name]
class BlockDecoder(object):

 @staticmethod
def _decode_block_string(block_string):

 assert isinstance(block_string, str)
 ops = block_string.split('_')
 options = {}
for op in ops:
splits = re.split(r'(\d.*)', op)
 if len(splits)>= 2:
key, value = splits[:2]
options[key] = value
assert(( 's' in options and len(options['s'])== 1)or 
(len(options['s'])== 2 and options['s'][0] == options['s'][1]))
return BlockArgs(
 kernel_size=int(options['k']),
num_repeat=int(options['r']),
input_filters=int(options['i']),
output_filters=int(options['o']),
expand_ratio=int(options['e']),
id_skip=('noskip' not in block_string),
se_ratio=float(options['se'])if 'se' in options else None,
stride=[int(options['s'][0])])
 @staticmethod
def _encode_block_string(block):

 args = [
'r%d' % block.num_repeat,
'k%d' % block.kernel_size,
's%d%d' %(block.strides[0], block.strides[1]),
'e%s' % block.expand_ratio,
'i%d' % block.input_filters,
'o%d' % block.output_filters
]
if 0 < block.se_ratio <= 1:
args.append('se%s' % block.se_ratio)
 if block.id_skip is False:
args.append('noskip')
 return '_'.join(args)
 @staticmethod
def decode(string_list):

 assert isinstance(string_list, list)
 blocks_args = []
for block_string in string_list:
blocks_args.append(BlockDecoder._decode_block_string(block_string))
return blocks_args
@staticmethod
def encode(blocks_args):

 block_strings = []
for block in blocks_args:
block_strings.append(BlockDecoder._encode_block_string(block))
return block_strings
def efficientnet(width_coefficient=None, depth_coefficient=None, dropout_rate=0.2,
drop_connect_rate=0.2, image_size=None, num_classes=1000):

 blocks_args = [
'r1_k3_s11_e1_i32_o16_se0.25', 'r2_k3_s22_e6_i16_o24_se0.25',
'r2_k5_s22_e6_i24_o40_se0.25', 'r3_k3_s22_e6_i40_o80_se0.25',
'r3_k5_s11_e6_i80_o112_se0.25', 'r4_k5_s22_e6_i112_o192_se0.25',
'r1_k3_s11_e6_i192_o320_se0.25',
]
blocks_args = BlockDecoder.decode(blocks_args)
 global_params = GlobalParams(
 batch_norm_momentum=0.99,
batch_norm_epsilon=1e-3,
dropout_rate=dropout_rate,
drop_connect_rate=drop_connect_rate,
num_classes=num_classes,
width_coefficient=width_coefficient,
depth_coefficient=depth_coefficient,
depth_divisor=8,
min_depth=None,
image_size=image_size, 
)
 return blocks_args, global_params
def get_model_params(model_name, override_params):

 if model_name.startswith('efficientnet'):
w, d, s, p = efficientnet_params(model_name)
 blocks_args, global_params = efficientnet(
 width_coefficient=w, depth_coefficient=d, dropout_rate=p, image_size=s)
 else:
raise NotImplementedError('model name is not pre-defined: %s' % model_name)
 if override_params:
global_params = global_params._replace(**override_params)
 return blocks_args, global_params
url_map = {
'efficientnet-b0': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/efficientnet-b0-355c32eb.pth',
'efficientnet-b1': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/efficientnet-b1-f1951068.pth',
'efficientnet-b2': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/efficientnet-b2-8bb594d6.pth',
'efficientnet-b3': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/efficientnet-b3-5fb5a3c3.pth',
'efficientnet-b4': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/efficientnet-b4-6ed6700e.pth',
'efficientnet-b5': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/efficientnet-b5-b6417697.pth',
'efficientnet-b6': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/efficientnet-b6-c76e70fd.pth',
'efficientnet-b7': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/efficientnet-b7-dcc49843.pth',
}
url_map_advprop = {
'efficientnet-b0': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/adv-efficientnet-b0-b64d5a18.pth',
'efficientnet-b1': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/adv-efficientnet-b1-0f3ce85a.pth',
'efficientnet-b2': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/adv-efficientnet-b2-6e9d97e5.pth',
'efficientnet-b3': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/adv-efficientnet-b3-cdd7c0f4.pth',
'efficientnet-b4': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/adv-efficientnet-b4-44fb3a87.pth',
'efficientnet-b5': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/adv-efficientnet-b5-86493f6b.pth',
'efficientnet-b6': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/adv-efficientnet-b6-ac80338e.pth',
'efficientnet-b7': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/adv-efficientnet-b7-4652b6dd.pth',
'efficientnet-b8': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/adv-efficientnet-b8-22a8fe65.pth',
}
def load_pretrained_weights(model, model_name, load_fc=True, advprop=False):

 url_map_ = url_map_advprop if advprop else url_map
state_dict = model_zoo.load_url(url_map_[model_name])
 if load_fc:
model.load_state_dict(state_dict)
 else:
state_dict.pop('_fc.weight')
 state_dict.pop('_fc.bias')
 res = model.load_state_dict(state_dict, strict=False)
 assert set(res.missing_keys)== set(['_fc.weight', '_fc.bias']), 'issue loading pretrained weights'
print('Loaded pretrained weights for {}'.format(model_name))<set_options>
data=pd.read_csv(".. /input/train.csv")
 data1=pd.read_csv(".. /input/test.csv" )
class MBConvBlock(nn.Module):

 def __init__(self, block_args, global_params):
super().__init__()
self._block_args = block_args
self._bn_mom = 1 - global_params.batch_norm_momentum
self._bn_eps = global_params.batch_norm_epsilon
self.has_se =(self._block_args.se_ratio is not None)and(0 < self._block_args.se_ratio <= 1)
 self.id_skip = block_args.id_skip
Conv2d = get_same_padding_conv2d(image_size=global_params.image_size)
 inp = self._block_args.input_filters
oup = self._block_args.input_filters * self._block_args.expand_ratio
if self._block_args.expand_ratio != 1:
self._expand_conv = Conv2d(in_channels=inp, out_channels=oup, kernel_size=1, bias=False)
 self._bn0 = nn.BatchNorm2d(num_features=oup, momentum=self._bn_mom, eps=self._bn_eps)
 k = self._block_args.kernel_size
s = self._block_args.stride
self._depthwise_conv = Conv2d(
 in_channels=oup, out_channels=oup, groups=oup,
kernel_size=k, stride=s, bias=False)
 self._bn1 = nn.BatchNorm2d(num_features=oup, momentum=self._bn_mom, eps=self._bn_eps)
 if self.has_se:
num_squeezed_channels = max(1, int(self._block_args.input_filters * self._block_args.se_ratio))
self._se_reduce = Conv2d(in_channels=oup, out_channels=num_squeezed_channels, kernel_size=1)
 self._se_expand = Conv2d(in_channels=num_squeezed_channels, out_channels=oup, kernel_size=1)
 final_oup = self._block_args.output_filters
self._project_conv = Conv2d(in_channels=oup, out_channels=final_oup, kernel_size=1, bias=False)
 self._bn2 = nn.BatchNorm2d(num_features=final_oup, momentum=self._bn_mom, eps=self._bn_eps)
 self._swish = MemoryEfficientSwish()
def forward(self, inputs, drop_connect_rate=None):

 x = inputs
if self._block_args.expand_ratio != 1:
x = self._swish(self._bn0(self._expand_conv(inputs)))
 x = self._swish(self._bn1(self._depthwise_conv(x)))
 if self.has_se:
x_squeezed = F.adaptive_avg_pool2d(x, 1)
 x_squeezed = self._se_expand(self._swish(self._se_reduce(x_squeezed)))
 x = torch.sigmoid(x_squeezed)* x
x = self._bn2(self._project_conv(x))
input_filters, output_filters = self._block_args.input_filters, self._block_args.output_filters
if self.id_skip and self._block_args.stride == 1 and input_filters == output_filters:
if drop_connect_rate:
x = drop_connect(x, p=drop_connect_rate, training=self.training)
 x = x + inputs
return x
def set_swish(self, memory_efficient=True):

 self._swish = MemoryEfficientSwish() if memory_efficient else Swish()
class EfficientNet(nn.Module):

 def __init__(self, blocks_args=None, global_params=None):
super().__init__()
assert isinstance(blocks_args, list), 'blocks_args should be a list'
assert len(blocks_args)> 0, 'block args must be greater than 0'
self._global_params = global_params
self._blocks_args = blocks_args
Conv2d = get_same_padding_conv2d(image_size=global_params.image_size)
 bn_mom = 1 - self._global_params.batch_norm_momentum
bn_eps = self._global_params.batch_norm_epsilon
in_channels = 3
out_channels = round_filters(32, self._global_params)
 self._conv_stem = Conv2d(in_channels, out_channels, kernel_size=3, stride=2, bias=False)
 self._bn0 = nn.BatchNorm2d(num_features=out_channels, momentum=bn_mom, eps=bn_eps)
 self._blocks = nn.ModuleList([])
 for block_args in self._blocks_args:
block_args = block_args._replace(
 input_filters=round_filters(block_args.input_filters, self._global_params),
output_filters=round_filters(block_args.output_filters, self._global_params),
num_repeat=round_repeats(block_args.num_repeat, self._global_params)
)
 self._blocks.append(MBConvBlock(block_args, self._global_params))
if block_args.num_repeat > 1:
block_args = block_args._replace(input_filters=block_args.output_filters, stride=1)
 for _ in range(block_args.num_repeat - 1):
self._blocks.append(MBConvBlock(block_args, self._global_params))
in_channels = block_args.output_filters
out_channels = round_filters(1280, self._global_params)
 self._conv_head = Conv2d(in_channels, out_channels, kernel_size=1, bias=False)
 self._bn1 = nn.BatchNorm2d(num_features=out_channels, momentum=bn_mom, eps=bn_eps)
 self._avg_pooling = nn.AdaptiveAvgPool2d(1)
 self._dropout = nn.Dropout(self._global_params.dropout_rate)
 self._fc = nn.Linear(out_channels, self._global_params.num_classes)
 self._swish = MemoryEfficientSwish()
def set_swish(self, memory_efficient=True):

 self._swish = MemoryEfficientSwish() if memory_efficient else Swish()
for block in self._blocks:
block.set_swish(memory_efficient)
 def extract_features(self, inputs):

 x = self._swish(self._bn0(self._conv_stem(inputs)))
 for idx, block in enumerate(self._blocks):
drop_connect_rate = self._global_params.drop_connect_rate
if drop_connect_rate:
drop_connect_rate *= float(idx)/ len(self._blocks)
 x = block(x, drop_connect_rate=drop_connect_rate)
 x = self._swish(self._bn1(self._conv_head(x)))
 return x
def forward(self, inputs):

 bs = inputs.size(0)
 x = self.extract_features(inputs)
 x = self._avg_pooling(x)
 x = x.view(bs, -1)
 x = self._dropout(x)
 x = self._fc(x)
 return x
@classmethod
def from_name(cls, model_name, override_params=None):
cls._check_model_name_is_valid(model_name)
 blocks_args, global_params = get_model_params(model_name, override_params)
 return cls(blocks_args, global_params)
 @classmethod
def from_pretrained(cls, model_name, advprop=False, num_classes=1000, in_channels=3):
model = cls.from_name(model_name, override_params={'num_classes': num_classes})
 load_pretrained_weights(model, model_name, load_fc=(num_classes == 1000), advprop=advprop)
 if in_channels != 3:
Conv2d = get_same_padding_conv2d(image_size = model._global_params.image_size)
 out_channels = round_filters(32, model._global_params)
 model._conv_stem = Conv2d(in_channels, out_channels, kernel_size=3, stride=2, bias=False)
 return model
@classmethod
def get_image_size(cls, model_name):
cls._check_model_name_is_valid(model_name)
 _, _, res, _ = efficientnet_params(model_name)
 return res
@classmethod
def _check_model_name_is_valid(cls, model_name):

 valid_models = ['efficientnet-b'+str(i)for i in range(9)]
if model_name not in valid_models:
raise ValueError('model_name should be one of: ' + ', '.join(valid_models))<normalization>
df = data[['epared1','epared2','epared3']]
x = df.stack()
data['epared'] = np.array(pd.Categorical(x[x!=0].index.get_level_values(1)))
 data['epared'] = data['epared'].apply(lambda x : 1 if x == 'epared1' else(2 if x == 'epared2' else 3))
df = data[['etecho1','etecho2','etecho3']]
x = df.stack()
data['etecho'] = np.array(pd.Categorical(x[x!=0].index.get_level_values(1)))
 data['etecho'] = data['epared'].apply(lambda x : 1 if x == 'etecho1' else(2 if x == 'etecho2' else 3))
df = data[['eviv1','eviv2','eviv3']]
x = df.stack()
data['eviv'] = np.array(pd.Categorical(x[x!=0].index.get_level_values(1)))
 data['eviv'] = data['epared'].apply(lambda x : 1 if x == 'eviv1' else(2 if x == 'eviv2' else 3))
data.drop(['epared1','epared2','epared3','etecho1','etecho2','etecho3','eviv1','eviv2','eviv3'],axis=1,inplace=True )
def gem(x, p=3, eps=1e-6):
return F.avg_pool2d(x.clamp(min=eps ).pow(p),(x.size(-2), x.size(-1)) ).pow(1./p)
 class GeM(nn.Module):
def __init__(self, p=3, eps=1e-6):
super(GeM,self ).__init__()
self.p = Parameter(torch.ones(1)*p)
 self.eps = eps
def forward(self, x):
return gem(x, p=self.p, eps=self.eps)
 def __repr__(self):
return self.__class__.__name__ + '(' + 'p=' + '{:.4f}'.format(self.p.data.tolist() [0])+ ', ' + 'eps=' + str(self.eps)+ ')'
class BirdModel(nn.Module):
def __init__(self, backbone, out_dim, gem_pooling=False):
super(BirdModel, self ).__init__()
self.gem_pooling = gem_pooling
self.backbone_name = backbone
if backbone.startswith('efficientnet'):
self.backbone = EfficientNet.from_name(backbone)
 self.last_linear = nn.Linear(self.backbone._fc.in_features, out_dim)
 else:
self.backbone = pretrainedmodels.__dict__[backbone](num_classes=1000, pretrained='imagenet')
 self.last_linear = nn.Linear(self.backbone.last_linear.in_features, out_dim)
 if self.gem_pooling:
if backbone.startswith('efficientnet'):
self.backbone._avg_pooling = GeM()
else:
self.backbone.avg_pool = GeM()
print('use gem pooling')
 def extract_seresnet(self, x):
x = self.backbone.features(x)
 x = self.backbone.avg_pool(x)
 if self.backbone.dropout is not None:
x = self.backbone.dropout(x)
 x = x.view(x.size(0), -1)
 return x
def extract_efficientnet(self, x):
bs = x.size(0)
 x = self.backbone.extract_features(x)
 x = self.backbone._avg_pooling(x)
 x = x.view(bs, -1)
 x = self.backbone._dropout(x)
 return x
def forward(self, x):
x = x.unsqueeze(1)
 x = x.expand(-1, 3, -1, -1)
 if self.backbone_name.startswith('efficientnet'):
x = self.extract_efficientnet(x)
 else:
x = extract_seresnet(x)
 x = self.last_linear(x)
 return {'logits':x} 
<define_variables>
df = data1[['epared1','epared2','epared3']]
x = df.stack()
data1['epared'] = np.array(pd.Categorical(x[x!=0].index.get_level_values(1)))
 data1['epared'] = data1['epared'].apply(lambda x : 1 if x == 'epared1' else(2 if x == 'epared2' else 3))
df = data1[['etecho1','etecho2','etecho3']]
x = df.stack()
data1['etecho'] = np.array(pd.Categorical(x[x!=0].index.get_level_values(1)))
 data1['etecho'] = data1['epared'].apply(lambda x : 1 if x == 'etecho1' else(2 if x == 'etecho2' else 3))
df = data1[['eviv1','eviv2','eviv3']]
x = df.stack()
data1['eviv'] = np.array(pd.Categorical(x[x!=0].index.get_level_values(1)))
 data1['eviv'] = data1['epared'].apply(lambda x : 1 if x == 'eviv1' else(2 if x == 'eviv2' else 3))
data1.drop(['epared1','epared2','epared3','etecho1','etecho2','etecho3','eviv1','eviv2','eviv3'],axis=1,inplace=True )
BIRD_CODE = {
'aldfly': 0, 'ameavo': 1, 'amebit': 2, 'amecro': 3, 'amegfi': 4,
'amekes': 5, 'amepip': 6, 'amered': 7, 'amerob': 8, 'amewig': 9,
'amewoo': 10, 'amtspa': 11, 'annhum': 12, 'astfly': 13, 'baisan': 14,
'baleag': 15, 'balori': 16, 'banswa': 17, 'barswa': 18, 'bawwar': 19,
'belkin1': 20, 'belspa2': 21, 'bewwre': 22, 'bkbcuc': 23, 'bkbmag1': 24,
'bkbwar': 25, 'bkcchi': 26, 'bkchum': 27, 'bkhgro': 28, 'bkpwar': 29,
'bktspa': 30, 'blkpho': 31, 'blugrb1': 32, 'blujay': 33, 'bnhcow': 34,
'boboli': 35, 'bongul': 36, 'brdowl': 37, 'brebla': 38, 'brespa': 39,
'brncre': 40, 'brnthr': 41, 'brthum': 42, 'brwhaw': 43, 'btbwar': 44,
'btnwar': 45, 'btywar': 46, 'buffle': 47, 'buggna': 48, 'buhvir': 49,
'bulori': 50, 'bushti': 51, 'buwtea': 52, 'buwwar': 53, 'cacwre': 54,
'calgul': 55, 'calqua': 56, 'camwar': 57, 'cangoo': 58, 'canwar': 59,
'canwre': 60, 'carwre': 61, 'casfin': 62, 'caster1': 63, 'casvir': 64,
'cedwax': 65, 'chispa': 66, 'chiswi': 67, 'chswar': 68, 'chukar': 69,
'clanut': 70, 'cliswa': 71, 'comgol': 72, 'comgra': 73, 'comloo': 74,
'commer': 75, 'comnig': 76, 'comrav': 77, 'comred': 78, 'comter': 79,
'comyel': 80, 'coohaw': 81, 'coshum': 82, 'cowscj1': 83, 'daejun': 84,
'doccor': 85, 'dowwoo': 86, 'dusfly': 87, 'eargre': 88, 'easblu': 89,
'easkin': 90, 'easmea': 91, 'easpho': 92, 'eastow': 93, 'eawpew': 94,
'eucdov': 95, 'eursta': 96, 'evegro': 97, 'fiespa': 98, 'fiscro': 99,
'foxspa': 100, 'gadwal': 101, 'gcrfin': 102, 'gnttow': 103, 'gnwtea': 104,
'gockin': 105, 'gocspa': 106, 'goleag': 107, 'grbher3': 108, 'grcfly': 109,
'greegr': 110, 'greroa': 111, 'greyel': 112, 'grhowl': 113, 'grnher': 114,
'grtgra': 115, 'grycat': 116, 'gryfly': 117, 'haiwoo': 118, 'hamfly': 119,
'hergul': 120, 'herthr': 121, 'hoomer': 122, 'hoowar': 123, 'horgre': 124,
'horlar': 125, 'houfin': 126, 'houspa': 127, 'houwre': 128, 'indbun': 129,
'juntit1': 130, 'killde': 131, 'labwoo': 132, 'larspa': 133, 'lazbun': 134,
'leabit': 135, 'leafly': 136, 'leasan': 137, 'lecthr': 138, 'lesgol': 139,
'lesnig': 140, 'lesyel': 141, 'lewwoo': 142, 'linspa': 143, 'lobcur': 144,
'lobdow': 145, 'logshr': 146, 'lotduc': 147, 'louwat': 148, 'macwar': 149,
'magwar': 150, 'mallar3': 151, 'marwre': 152, 'merlin': 153, 'moublu': 154,
'mouchi': 155, 'moudov': 156, 'norcar': 157, 'norfli': 158, 'norhar2': 159,
'normoc': 160, 'norpar': 161, 'norpin': 162, 'norsho': 163, 'norwat': 164,
'nrwswa': 165, 'nutwoo': 166, 'olsfly': 167, 'orcwar': 168, 'osprey': 169,
'ovenbi1': 170, 'palwar': 171, 'pasfly': 172, 'pecsan': 173, 'perfal': 174,
'phaino': 175, 'pibgre': 176, 'pilwoo': 177, 'pingro': 178, 'pinjay': 179,
'pinsis': 180, 'pinwar': 181, 'plsvir': 182, 'prawar': 183, 'purfin': 184,
'pygnut': 185, 'rebmer': 186, 'rebnut': 187, 'rebsap': 188, 'rebwoo': 189,
'redcro': 190, 'redhea': 191, 'reevir1': 192, 'renpha': 193, 'reshaw': 194,
'rethaw': 195, 'rewbla': 196, 'ribgul': 197, 'rinduc': 198, 'robgro': 199,
'rocpig': 200, 'rocwre': 201, 'rthhum': 202, 'ruckin': 203, 'rudduc': 204,
'rufgro': 205, 'rufhum': 206, 'rusbla': 207, 'sagspa1': 208, 'sagthr': 209,
'savspa': 210, 'saypho': 211, 'scatan': 212, 'scoori': 213, 'semplo': 214,
'semsan': 215, 'sheowl': 216, 'shshaw': 217, 'snobun': 218, 'snogoo': 219,
'solsan': 220, 'sonspa': 221, 'sora': 222, 'sposan': 223, 'spotow': 224,
'stejay': 225, 'swahaw': 226, 'swaspa': 227, 'swathr': 228, 'treswa': 229,
'truswa': 230, 'tuftit': 231, 'tunswa': 232, 'veery': 233, 'vesspa': 234,
'vigswa': 235, 'warvir': 236, 'wesblu': 237, 'wesgre': 238, 'weskin': 239,
'wesmea': 240, 'wessan': 241, 'westan': 242, 'wewpew': 243, 'whbnut': 244,
'whcspa': 245, 'whfibi': 246, 'whtspa': 247, 'whtswi': 248, 'wilfly': 249,
'wilsni1': 250, 'wiltur': 251, 'winwre3': 252, 'wlswar': 253, 'wooduc': 254,
'wooscj2': 255, 'woothr': 256, 'y00475': 257, 'yebfly': 258, 'yebsap': 259,
'yehbla': 260, 'yelwar': 261, 'yerwar': 262, 'yetvir': 263, 'nocall':264
}
INV_BIRD_CODE = {v: k for k, v in BIRD_CODE.items() }<load_pretrained>
def data_cleaning(data):
data['dependency']=np.sqrt(data['SQBdependency'])
 data['rez_esc']=data['rez_esc'].fillna(0)
 data['v18q1']=data['v18q1'].fillna(0)
 data['v2a1']=data['v2a1'].fillna(0)
 conditions = [ 
(data['edjefe']=='no')&(data['edjefa']=='no'), 
(data['edjefe']=='yes')&(data['edjefa']=='no'), 
(data['edjefe']=='no')&(data['edjefa']=='yes'), 
(data['edjefe']!='no')&(data['edjefe']!='yes')&(data['edjefa']=='no'), 
(data['edjefe']=='no')&(data['edjefa']!='no')
 ]
choices = [0, 1, 1, data['edjefe'], data['edjefa']]
data['edjefx']=np.select(conditions, choices)
 data['edjefx']=data['edjefx'].astype(int)
 data.drop(['edjefe', 'edjefa'], axis=1, inplace=True)
 meaneduc_nan=data[data['meaneduc'].isnull() ][['Id','idhogar','escolari']]
me=meaneduc_nan.groupby('idhogar')['escolari'].mean().reset_index()
for row in meaneduc_nan.iterrows() :
idx=row[0]
idhogar=row[1]['idhogar']
m=me[me['idhogar']==idhogar]['escolari'].tolist() [0]
data.at[idx, 'meaneduc']=m
data.at[idx, 'SQBmeaned']=m*m
return data
def load_checkpoint(backbone, epoch, fold, seed, fname):
model = BirdModel(backbone, out_dim=len(BIRD_CODE), gem_pooling=False)
 checkpoint = torch.load('/kaggle/input/cpmp-birds-210/%s_%d_%d_%d.pt' %(fname, fold, seed, epoch))
model.load_state_dict(checkpoint['model'])
 model.cuda()
model.eval()
return model<load_pretrained>
total = data.isnull().sum().sort_values(ascending=False)
 percent =(data.isnull().sum() /data.isnull().count() ).sort_values(ascending=False)
 missing_data = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
 missing_data.head(20 )
backbone='efficientnet-b3'
model_name = 'model_210'
models = [load_checkpoint(backbone, 59, fold, 0, model_name)for fold in range(5)]<define_variables>
data1.loc[data1['rez_esc'] == 99.0 , 'rez_esc'] = 5
TEST = Path(".. /input/birdsong-recognition/test_audio" ).exists()<load_from_csv>
data['roof_waste_material'] = np.nan
data1['roof_waste_material'] = np.nan
data['electricity_other'] = np.nan
data1['electricity_other'] = np.nan
def fill_roof_exception(x):
if(x['techozinc'] == 0)and(x['techoentrepiso'] == 0)and(x['techocane'] == 0)and(x['techootro'] == 0):
return 1
else:
return 0
def fill_no_electricity(x):
if(x['public'] == 0)and(x['planpri'] == 0)and(x['noelec'] == 0)and(x['coopele'] == 0):
return 1
else:
return 0
data['roof_waste_material'] = data.apply(lambda x : fill_roof_exception(x),axis=1)
 data1['roof_waste_material'] = data1.apply(lambda x : fill_roof_exception(x),axis=1)
 data['electricity_other'] = data.apply(lambda x : fill_no_electricity(x),axis=1)
 data1['electricity_other'] = data1.apply(lambda x : fill_no_electricity(x),axis=1 )
if TEST:
DATA_DIR = Path(".. /input/birdsong-recognition/")
 else:
DATA_DIR = Path(".. /input/birdcall-check/")
 test = pd.read_csv(DATA_DIR / "test.csv")
 test_audio = DATA_DIR / "test_audio"
test.head()<feature_engineering>
def feature_engineering(train_set):
train_set['adult'] = train_set['hogar_adul'] - train_set['hogar_mayor']
train_set['dependency_count'] = train_set['hogar_nin'] + train_set['hogar_mayor']
train_set['child_percent'] = train_set['hogar_nin']/train_set['hogar_total']*100
train_set['elder_percent'] = train_set['hogar_mayor']/train_set['hogar_total']*100
train_set['adult_percent'] = train_set['hogar_adul']/train_set['hogar_total']*100
train_set['rent_per_person'] = train_set['v2a1']/train_set['hhsize']
train_set['no_appliances'] = train_set['refrig'] + train_set['computer'] + train_set['television']
train_set['rent_per_room'] = train_set['v2a1']/train_set['rooms']
train_set['room_per_person_household'] = train_set['hhsize']/train_set['rooms']
train_set['escolari_age'] = train_set['escolari']/train_set['age']
train_set['rez_esc_escolari'] = train_set['rez_esc']/train_set['escolari']
return train_set
PERIOD = 5
def get_melspec(clip, sr, period):
length = len(clip)
 if period > length:
start = np.random.randint(period - length)
 tmp = np.zeros(period, dtype=clip.dtype)
 tmp[start : start + length] = clip
clip = tmp
n_fft = 1024
win_length = n_fft
hop_length = int(( len(clip)- win_length + n_fft)/ IMAGE_WIDTH)+ 1
spect = np.abs(librosa.stft(y=clip, n_fft=n_fft, hop_length=hop_length, win_length=win_length))
if spect.shape[1] < IMAGE_WIDTH:
hop_length = hop_length - 1
spect = np.abs(librosa.stft(y=clip, n_fft=n_fft, hop_length=hop_length, win_length=win_length))
if spect.shape[1] > IMAGE_WIDTH:
spect = spect[:, :IMAGE_WIDTH]
n_mels = IMAGE_HEIGHT // 2
spect = np.square(spect)
 spect = librosa.feature.melspectrogram(S=spect, sr=sr, n_fft=n_fft, n_mels=n_mels, fmin=300, fmax=16000)
 spect = librosa.power_to_db(spect)
 spect = resize(spect,(IMAGE_HEIGHT, IMAGE_WIDTH), preserve_range=True, anti_aliasing=True)
 spect = spect - spect.min()
smax = spect.max()
if smax >= 0.001:
spect = spect / smax
else:
spect[...] = 0
return spect<import_modules>
feature_engineering(data)
 feature_engineering(data1 )
class BirdTestDataset(Dataset):
def __init__(self,
audio_id,
data_path: Path,
period=PERIOD 
):
super(BirdTestDataset, self ).__init__()
self.audio_id = audio_id
self.data_path = data_path
clip, sr = get_clip_sr(audio_id, data_path)
 self.clip = clip
self.sr = sr
period = period * sr
self.period = period
self.starts = list(range(0, int(np.round(clip.shape[0] /(period)))- 1))
def __len__(self):
return len(self.starts)
 def __getitem__(self, idx: int):
clip = self.clip[idx * self.period :(idx +1)* self.period]
melspec = get_melspec(clip, self.sr, self.period)
 return {
"image": torch.from_numpy(melspec),
}<predict_on_test>
df_train = pd.DataFrame()
df_test = pd.DataFrame()
other_list = ['escolari', 'age', 'escolari_age']
for item in other_list:
for function in ['mean','std','min','max','sum']:
group_train = data[item].groupby(data['idhogar'] ).agg(function)
 group_test = data1[item].groupby(data1['idhogar'] ).agg(function)
 new_col = item + '_' + function
df_train[new_col] = group_train
df_test[new_col] = group_test
def predict_audio(loader, models, site):
ALL_LOGITS = []
with torch.no_grad() :
for batch in(loader):
data = batch['image'].cuda()
for(j, model)in enumerate(models):
out = model(data)
 logits = out['logits'].detach()
if j == 0:
LOGITS = logits
else:
LOGITS =(LOGITS + logits ).detach()
LOGITS =(LOGITS/j ).detach()
ALL_LOGITS.append(LOGITS)
 ALL_LOGITS = torch.cat(ALL_LOGITS ).cpu().numpy()
if site == 'site_3':
ALL_LOGITS = np.max(ALL_LOGITS, axis=0, keepdims=True)
 return ALL_LOGITS<normalization>
df_test = df_test.reset_index()
df_train = df_train.reset_index()
data = pd.merge(data, df_train, on='idhogar')
 data1 = pd.merge(data1, df_test, on='idhogar' )
def get_clip_sr(audio_id, data_path):
path = data_path /(audio_id + '.mp3')
 clip, sr_native = librosa.core.audio.__audioread_load(path, offset=0.0, duration=None, dtype=np.float32)
 clip = librosa.to_mono(clip)
 sr = 32000
if sr_native > 0:
clip = librosa.resample(clip, sr_native, sr, res_type='kaiser_fast')
 return clip, sr<load_from_csv>
train = pd.read_csv('.. /input/birdsong-recognition/train.csv' )<define_variables>
data.fillna(0, inplace=True)
 data1.fillna(0, inplace=True)
 data = data.replace({np.inf: 0})
 data1 = data1.replace({np.inf: 0} )
def get_code(logits, thr):
codes = [INV_BIRD_CODE[i] for i in range(len(logits)) if logits[i] >= thr]
if codes == []:
codes = 'nocall'
else:
codes = ' '.join(codes)
 return codes 
<groupby>
total = data.isnull().sum().sort_values(ascending=False)
 percent =(data.isnull().sum() /data.isnull().count() ).sort_values(ascending=False)
 missing_data = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
 missing_data.head(20 )
df = test.groupby('audio_id' ).first().reset_index()
df<import_modules>
from scipy.special import expit, logit<create_dataframe>
print([column for column in data1.columns if data1[column].isnull().sum() >0])
thr = logit(0.7)
 BATCH_SIZE = 32
WORKERS=2
IMAGE_HEIGHT = 300
IMAGE_WIDTH = 600
sub = test.copy()
sub = test.sort_values(by=['audio_id', 'seconds'] ).reset_index(drop=True)
 df = sub.groupby('audio_id' ).first().reset_index()
for site, audio_id in zip(df.site,(df.audio_id)) :
dataset = BirdTestDataset(audio_id, test_audio)
 loader = DataLoader(
 dataset,
batch_size=BATCH_SIZE,
num_workers=WORKERS,
shuffle=False,
pin_memory=True, 
)
 logits = predict_audio(loader, models, site)
 preds = [get_code(l, thr)for l in logits]
len_data = len(sub.loc[sub.audio_id == audio_id])
 if len(preds)< len_data:
preds.extend(['nocall' *(len_data - len(preds)) ])
 preds = preds[:len_data]
sub.loc[sub.audio_id == audio_id, 'birds'] = preds
sub<save_to_csv>
train_set, test_set = train_test_split(data, test_size = 0.1, random_state = 42)
 data = train_set
valid_data = test_set
sub[['row_id', 'birds']].to_csv("submission.csv", index=False )<import_modules>
valid_data=valid_data.drop(['Id','idhogar'],1)
 y = valid_data.Target.tolist()
valid_data = valid_data.drop('Target', 1)
 X = np.array(valid_data )
import cv2
import audioread
import logging
import os
import random
import time
import warnings
import librosa
import librosa.display as display
import numpy as np
import pandas as pd
import soundfile as sf
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
import torch.utils.data as data
from contextlib import contextmanager
from IPython.display import Audio
from pathlib import Path
from typing import Optional, List
from catalyst.dl import SupervisedRunner, State, CallbackOrder, Callback, CheckpointCallback
from fastprogress import progress_bar
from sklearn.model_selection import StratifiedKFold
from sklearn.metrics import f1_score, average_precision_score<set_options>
data=data.drop(['Id','idhogar'],1 )
def set_seed(seed: int = 42):
random.seed(seed)
 np.random.seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
def get_logger(out_file=None):
logger = logging.getLogger()
formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s")
 logger.handlers = []
logger.setLevel(logging.INFO)
 handler = logging.StreamHandler()
handler.setFormatter(formatter)
 handler.setLevel(logging.INFO)
 logger.addHandler(handler)
 if out_file is not None:
fh = logging.FileHandler(out_file)
 fh.setFormatter(formatter)
 fh.setLevel(logging.INFO)
 logger.addHandler(fh)
 logger.info("logger set up")
 return logger
@contextmanager
def timer(name: str, logger: Optional[logging.Logger] = None):
t0 = time.time()
msg = f"[{name}] start"
if logger is None:
print(msg)
 else:
logger.info(msg)
 yield
msg = f"[{name}] done in {time.time() - t0:.2f} s"
if logger is None:
print(msg)
 else:
logger.info(msg)
 set_seed(1213 )<define_variables>
y = np.array(data.Target.tolist())
 data = data.drop('Target', 1)
 X = np.array(data.as_matrix() )
ROOT = Path.cwd().parent
INPUT_ROOT = ROOT / "input"
RAW_DATA = INPUT_ROOT / "birdsong-recognition"
TRAIN_AUDIO_DIR = RAW_DATA / "train_audio"
TRAIN_RESAMPLED_AUDIO_DIRS = [
INPUT_ROOT / "birdsong-resampled-train-audio-{:0>2}".format(i)for i in range(5)
 ]
TEST_AUDIO_DIR = RAW_DATA / "test_audio"<load_from_csv>
skf = StratifiedKFold(n_splits=5 ,shuffle = True, random_state = 42)
 for train_index, test_index in skf.split(X, y):
X_train, y_train = X[train_index], y[train_index]
X_test, y_test = X[test_index], y[test_index]
train = pd.read_csv(TRAIN_RESAMPLED_AUDIO_DIRS[0] / "train_mod.csv")
 if not TEST_AUDIO_DIR.exists() :
TEST_AUDIO_DIR = INPUT_ROOT / "birdcall-check" / "test_audio"
test = pd.read_csv(INPUT_ROOT / "birdcall-check" / "test.csv")
 else:
test = pd.read_csv(RAW_DATA / "test.csv" )<categorify>
from sklearn.model_selection import GridSearchCV
class DropStripes(nn.Module):
def __init__(self, dim, drop_width, stripes_num):

 super(DropStripes, self ).__init__()
assert dim in [2, 3]
self.dim = dim
self.drop_width = drop_width
self.stripes_num = stripes_num
def forward(self, input):

 assert input.ndimension() == 4
if self.training is False:
return input
else:
batch_size = input.shape[0]
total_width = input.shape[self.dim]
for n in range(batch_size):
self.transform_slice(input[n], total_width)
 return input
def transform_slice(self, e, total_width):

 for _ in range(self.stripes_num):
distance = torch.randint(low=0, high=self.drop_width, size=(1,)) [0]
bgn = torch.randint(low=0, high=total_width - distance, size=(1,)) [0]
if self.dim == 2:
e[:, bgn : bgn + distance, :] = 0
elif self.dim == 3:
e[:, :, bgn : bgn + distance] = 0
class SpecAugmentation(nn.Module):
def __init__(self, time_drop_width, time_stripes_num, freq_drop_width,
freq_stripes_num):

 super(SpecAugmentation, self ).__init__()
self.time_dropper = DropStripes(dim=2, drop_width=time_drop_width,
stripes_num=time_stripes_num)
 self.freq_dropper = DropStripes(dim=3, drop_width=freq_drop_width,
stripes_num=freq_stripes_num)
 def forward(self, input):
x = self.time_dropper(input)
 x = self.freq_dropper(x)
 return x<train_model>
def init_layer(layer):
nn.init.xavier_uniform_(layer.weight)
 if hasattr(layer, "bias"):
if layer.bias is not None:
layer.bias.data.fill_(0.)
def init_bn(bn):
bn.bias.data.fill_(0.)
bn.weight.data.fill_(1.0)
 def interpolate(x: torch.Tensor, ratio: int):

(batch_size, time_steps, classes_num)= x.shape
upsampled = x[:, :, None, :].repeat(1, 1, ratio, 1)
 upsampled = upsampled.reshape(batch_size, time_steps * ratio, classes_num)
 return upsampled
def pad_framewise_output(framewise_output: torch.Tensor, frames_num: int):

 pad = framewise_output[:, -1:, :].repeat(
 1, frames_num - framewise_output.shape[1], 1)

output = torch.cat(( framewise_output, pad), dim=1)

return output
class ConvBlock(nn.Module):
def __init__(self, in_channels: int, out_channels: int):
super().__init__()
self.conv1 = nn.Conv2d(
 in_channels=in_channels,
out_channels=out_channels,
kernel_size=(3, 3),
stride=(1, 1),
padding=(1, 1),
bias=False)
 self.conv2 = nn.Conv2d(
 in_channels=out_channels,
out_channels=out_channels,
kernel_size=(3, 3),
stride=(1, 1),
padding=(1, 1),
bias=False)
 self.bn1 = nn.BatchNorm2d(out_channels)
 self.bn2 = nn.BatchNorm2d(out_channels)
 self.init_weight()
def init_weight(self):
init_layer(self.conv1)
 init_layer(self.conv2)
 init_bn(self.bn1)
 init_bn(self.bn2)
 def forward(self, input, pool_size=(2, 2), pool_type='avg'):
x = input
x = F.relu_(self.bn1(self.conv1(x)))
 x = F.relu_(self.bn2(self.conv2(x)))
 if pool_type == 'max':
x = F.max_pool2d(x, kernel_size=pool_size)
 elif pool_type == 'avg':
x = F.avg_pool2d(x, kernel_size=pool_size)
 elif pool_type == 'avg+max':
x1 = F.avg_pool2d(x, kernel_size=pool_size)
 x2 = F.max_pool2d(x, kernel_size=pool_size)
 x = x1 + x2
else:
raise Exception('Incorrect argument!')
 return x
class AttBlock(nn.Module):
def __init__(self,
in_features: int,
out_features: int,
activation="linear",
temperature=1.0):
super().__init__()
self.activation = activation
self.temperature = temperature
self.att = nn.Conv1d(
 in_channels=in_features,
out_channels=out_features,
kernel_size=1,
stride=1,
padding=0,
bias=True)
 self.cla = nn.Conv1d(
 in_channels=in_features,
out_channels=out_features,
kernel_size=1,
stride=1,
padding=0,
bias=True)
 self.bn_att = nn.BatchNorm1d(out_features)
 self.init_weights()
def init_weights(self):
init_layer(self.att)
 init_layer(self.cla)
 init_bn(self.bn_att)
 def forward(self, x):
norm_att = torch.softmax(torch.clamp(self.att(x), -10, 10), dim=-1)
 cla = self.nonlinear_transform(self.cla(x))
x = torch.sum(norm_att * cla, dim=2)
 return x, norm_att, cla
def nonlinear_transform(self, x):
if self.activation == 'linear':
return x
elif self.activation == 'sigmoid':
return torch.sigmoid(x )<set_options>
lgb_cfl = lgb.LGBMClassifier(max_depth=-1, learning_rate=0.1, objective='multiclass',
random_state=None, silent=True, metric='None',
n_jobs=4, n_estimators=5000, class_weight='balanced',
colsample_bytree = 0.89, min_child_samples = 30, num_leaves = 32, subsample = 0.96)
 lgb_cfl.fit(X_train, y_train)
 y_pred = lgb_cfl.predict(X_test)
 y_score = lgb_cfl.predict_proba(X_test)[:,1]
cm = confusion_matrix(y_test, y_pred)
 class_names = [1,2,3,4]
plt.figure()
plot_confusion_matrix(cm,
classes = class_names,
title = 'LGB Confusion matrix')
 plt.savefig('2.lgb_cfl_confusion_matrix.png')
 plt.show()
f1_score(y_test,y_pred, labels=None, pos_label=1, average= 'macro', sample_weight=None )
class PANNsCNN14Att(nn.Module):
def __init__(self, sample_rate: int, window_size: int, hop_size: int,
mel_bins: int, fmin: int, fmax: int, classes_num: int):
super().__init__()
window = 'hann'
center = True
pad_mode = 'reflect'
ref = 1.0
amin = 1e-10
top_db = None
self.interpolate_ratio = 32
self.spectrogram_extractor = Spectrogram(
 n_fft=window_size,
hop_length=hop_size,
win_length=window_size,
window=window,
center=center,
pad_mode=pad_mode,
freeze_parameters=True)
 self.logmel_extractor = LogmelFilterBank(
 sr=sample_rate,
n_fft=window_size,
n_mels=mel_bins,
fmin=fmin,
fmax=fmax,
ref=ref,
amin=amin,
top_db=top_db,
freeze_parameters=True)
 self.spec_augmenter = SpecAugmentation(
 time_drop_width=64,
time_stripes_num=2,
freq_drop_width=8,
freq_stripes_num=2)
 self.bn0 = nn.BatchNorm2d(mel_bins)
 self.conv_block1 = ConvBlock(in_channels=1, out_channels=64)
 self.conv_block2 = ConvBlock(in_channels=64, out_channels=128)
 self.conv_block3 = ConvBlock(in_channels=128, out_channels=256)
 self.conv_block4 = ConvBlock(in_channels=256, out_channels=512)
 self.conv_block5 = ConvBlock(in_channels=512, out_channels=1024)
 self.conv_block6 = ConvBlock(in_channels=1024, out_channels=2048)
 self.fc1 = nn.Linear(2048, 2048, bias=True)
 self.att_block = AttBlock(2048, classes_num, activation='sigmoid')
 self.init_weight()
def init_weight(self):
init_bn(self.bn0)
 init_layer(self.fc1)
 def cnn_feature_extractor(self, x):
x = self.conv_block1(x, pool_size=(2, 2), pool_type='avg')
 x = F.dropout(x, p=0.2, training=self.training)
 x = self.conv_block2(x, pool_size=(2, 2), pool_type='avg')
 x = F.dropout(x, p=0.2, training=self.training)
 x = self.conv_block3(x, pool_size=(2, 2), pool_type='avg')
 x = F.dropout(x, p=0.2, training=self.training)
 x = self.conv_block4(x, pool_size=(2, 2), pool_type='avg')
 x = F.dropout(x, p=0.2, training=self.training)
 x = self.conv_block5(x, pool_size=(2, 2), pool_type='avg')
 x = F.dropout(x, p=0.2, training=self.training)
 x = self.conv_block6(x, pool_size=(1, 1), pool_type='avg')
 x = F.dropout(x, p=0.2, training=self.training)
 return x
def preprocess(self, input, mixup_lambda=None):
x = self.spectrogram_extractor(input)
 x = self.logmel_extractor(x)
 frames_num = x.shape[2]
x = x.transpose(1, 3)
 x = self.bn0(x)
 x = x.transpose(1, 3)
 if self.training:
x = self.spec_augmenter(x)
 if self.training and mixup_lambda is not None:
x = do_mixup(x, mixup_lambda)
 return x, frames_num
def forward(self, input, mixup_lambda=None):

 x, frames_num = self.preprocess(input, mixup_lambda=mixup_lambda)
 x = self.cnn_feature_extractor(x)
 x = torch.mean(x, dim=3)
 x1 = F.max_pool1d(x, kernel_size=3, stride=1, padding=1)
 x2 = F.avg_pool1d(x, kernel_size=3, stride=1, padding=1)
 x = x1 + x2
x = F.dropout(x, p=0.5, training=self.training)
 x = x.transpose(1, 2)
 x = F.relu_(self.fc1(x))
x = x.transpose(1, 2)
 x = F.dropout(x, p=0.5, training=self.training)
(clipwise_output, norm_att, segmentwise_output)= self.att_block(x)
 segmentwise_output = segmentwise_output.transpose(1, 2)
 framewise_output = interpolate(segmentwise_output,
self.interpolate_ratio)
 framewise_output = pad_framewise_output(framewise_output, frames_num)
 output_dict = {
'framewise_output': framewise_output,
'clipwise_output': clipwise_output
}
return output_dict<prepare_x_and_y>
SR = 32000
y, _ = librosa.load(TRAIN_RESAMPLED_AUDIO_DIRS[0] / "aldfly" / "XC134874.wav",
sr=SR,
res_type="kaiser_fast",
mono=True)
<choose_model_class>
model_config = {
"sample_rate": 32000,
"window_size": 1024,
"hop_size": 320,
"mel_bins": 64,
"fmin": 20,
"fmax": 16000,
"classes_num": 264
}
model = PANNsCNN14Att(**model_config )<predict_on_test>
melspec_numpy = melspec.detach().numpy() [0, 0].transpose(1, 0)
 display.specshow(melspec_numpy, sr=SR, y_axis="mel");<define_variables>
BIRD_CODE = {
'aldfly': 0, 'ameavo': 1, 'amebit': 2, 'amecro': 3, 'amegfi': 4,
'amekes': 5, 'amepip': 6, 'amered': 7, 'amerob': 8, 'amewig': 9,
'amewoo': 10, 'amtspa': 11, 'annhum': 12, 'astfly': 13, 'baisan': 14,
'baleag': 15, 'balori': 16, 'banswa': 17, 'barswa': 18, 'bawwar': 19,
'belkin1': 20, 'belspa2': 21, 'bewwre': 22, 'bkbcuc': 23, 'bkbmag1': 24,
'bkbwar': 25, 'bkcchi': 26, 'bkchum': 27, 'bkhgro': 28, 'bkpwar': 29,
'bktspa': 30, 'blkpho': 31, 'blugrb1': 32, 'blujay': 33, 'bnhcow': 34,
'boboli': 35, 'bongul': 36, 'brdowl': 37, 'brebla': 38, 'brespa': 39,
'brncre': 40, 'brnthr': 41, 'brthum': 42, 'brwhaw': 43, 'btbwar': 44,
'btnwar': 45, 'btywar': 46, 'buffle': 47, 'buggna': 48, 'buhvir': 49,
'bulori': 50, 'bushti': 51, 'buwtea': 52, 'buwwar': 53, 'cacwre': 54,
'calgul': 55, 'calqua': 56, 'camwar': 57, 'cangoo': 58, 'canwar': 59,
'canwre': 60, 'carwre': 61, 'casfin': 62, 'caster1': 63, 'casvir': 64,
'cedwax': 65, 'chispa': 66, 'chiswi': 67, 'chswar': 68, 'chukar': 69,
'clanut': 70, 'cliswa': 71, 'comgol': 72, 'comgra': 73, 'comloo': 74,
'commer': 75, 'comnig': 76, 'comrav': 77, 'comred': 78, 'comter': 79,
'comyel': 80, 'coohaw': 81, 'coshum': 82, 'cowscj1': 83, 'daejun': 84,
'doccor': 85, 'dowwoo': 86, 'dusfly': 87, 'eargre': 88, 'easblu': 89,
'easkin': 90, 'easmea': 91, 'easpho': 92, 'eastow': 93, 'eawpew': 94,
'eucdov': 95, 'eursta': 96, 'evegro': 97, 'fiespa': 98, 'fiscro': 99,
'foxspa': 100, 'gadwal': 101, 'gcrfin': 102, 'gnttow': 103, 'gnwtea': 104,
'gockin': 105, 'gocspa': 106, 'goleag': 107, 'grbher3': 108, 'grcfly': 109,
'greegr': 110, 'greroa': 111, 'greyel': 112, 'grhowl': 113, 'grnher': 114,
'grtgra': 115, 'grycat': 116, 'gryfly': 117, 'haiwoo': 118, 'hamfly': 119,
'hergul': 120, 'herthr': 121, 'hoomer': 122, 'hoowar': 123, 'horgre': 124,
'horlar': 125, 'houfin': 126, 'houspa': 127, 'houwre': 128, 'indbun': 129,
'juntit1': 130, 'killde': 131, 'labwoo': 132, 'larspa': 133, 'lazbun': 134,
'leabit': 135, 'leafly': 136, 'leasan': 137, 'lecthr': 138, 'lesgol': 139,
'lesnig': 140, 'lesyel': 141, 'lewwoo': 142, 'linspa': 143, 'lobcur': 144,
'lobdow': 145, 'logshr': 146, 'lotduc': 147, 'louwat': 148, 'macwar': 149,
'magwar': 150, 'mallar3': 151, 'marwre': 152, 'merlin': 153, 'moublu': 154,
'mouchi': 155, 'moudov': 156, 'norcar': 157, 'norfli': 158, 'norhar2': 159,
'normoc': 160, 'norpar': 161, 'norpin': 162, 'norsho': 163, 'norwat': 164,
'nrwswa': 165, 'nutwoo': 166, 'olsfly': 167, 'orcwar': 168, 'osprey': 169,
'ovenbi1': 170, 'palwar': 171, 'pasfly': 172, 'pecsan': 173, 'perfal': 174,
'phaino': 175, 'pibgre': 176, 'pilwoo': 177, 'pingro': 178, 'pinjay': 179,
'pinsis': 180, 'pinwar': 181, 'plsvir': 182, 'prawar': 183, 'purfin': 184,
'pygnut': 185, 'rebmer': 186, 'rebnut': 187, 'rebsap': 188, 'rebwoo': 189,
'redcro': 190, 'redhea': 191, 'reevir1': 192, 'renpha': 193, 'reshaw': 194,
'rethaw': 195, 'rewbla': 196, 'ribgul': 197, 'rinduc': 198, 'robgro': 199,
'rocpig': 200, 'rocwre': 201, 'rthhum': 202, 'ruckin': 203, 'rudduc': 204,
'rufgro': 205, 'rufhum': 206, 'rusbla': 207, 'sagspa1': 208, 'sagthr': 209,
'savspa': 210, 'saypho': 211, 'scatan': 212, 'scoori': 213, 'semplo': 214,
'semsan': 215, 'sheowl': 216, 'shshaw': 217, 'snobun': 218, 'snogoo': 219,
'solsan': 220, 'sonspa': 221, 'sora': 222, 'sposan': 223, 'spotow': 224,
'stejay': 225, 'swahaw': 226, 'swaspa': 227, 'swathr': 228, 'treswa': 229,
'truswa': 230, 'tuftit': 231, 'tunswa': 232, 'veery': 233, 'vesspa': 234,
'vigswa': 235, 'warvir': 236, 'wesblu': 237, 'wesgre': 238, 'weskin': 239,
'wesmea': 240, 'wessan': 241, 'westan': 242, 'wewpew': 243, 'whbnut': 244,
'whcspa': 245, 'whfibi': 246, 'whtspa': 247, 'whtswi': 248, 'wilfly': 249,
'wilsni1': 250, 'wiltur': 251, 'winwre3': 252, 'wlswar': 253, 'wooduc': 254,
'wooscj2': 255, 'woothr': 256, 'y00475': 257, 'yebfly': 258, 'yebsap': 259,
'yehbla': 260, 'yelwar': 261, 'yerwar': 262, 'yetvir': 263
}
INV_BIRD_CODE = {v: k for k, v in BIRD_CODE.items() }<prepare_x_and_y>
f1_score(y_test,y_pred, labels=None, pos_label=1, average= 'macro', sample_weight=None )
PERIOD = 10
class PANNsDataset(data.Dataset):
def __init__(
 self,
file_list: List[List[str]],
waveform_transforms=None):
self.file_list = file_list
self.waveform_transforms = waveform_transforms
def __len__(self):
return len(self.file_list)
 def __getitem__(self, idx: int):
wav_path, ebird_code = self.file_list[idx]
y, sr = sf.read(wav_path)
 if self.waveform_transforms:
y = self.waveform_transforms(y)
 else:
len_y = len(y)
 effective_length = sr * PERIOD
if len_y < effective_length:
new_y = np.zeros(effective_length, dtype=y.dtype)
 start = np.random.randint(effective_length - len_y)
 new_y[start:start + len_y] = y
y = new_y.astype(np.float32)
 elif len_y > effective_length:
start = np.random.randint(len_y - effective_length)
 y = y[start:start + effective_length].astype(np.float32)
 else:
y = y.astype(np.float32)
 labels = np.zeros(len(BIRD_CODE), dtype="f")
 for b_code in ebird_code:
labels[BIRD_CODE[b_code]] = 1
return {"waveform": y, "targets": labels}<normalization>
Target=voting_cfl.predict(data1 )
class PANNsLoss(nn.Module):
def __init__(self):
super().__init__()
self.bce = nn.BCELoss()
def forward(self, input, target):
input_ = input["clipwise_output"]
input_ = torch.where(torch.isnan(input_),
torch.zeros_like(input_),
input_)
 input_ = torch.where(torch.isinf(input_),
torch.zeros_like(input_),
input_)
 target = target.float()
input_ = torch.clamp(input_, 0.0, 1.0)
 return self.bce(input_, target )<load_pretrained>
Id=pd.DataFrame(Id )
class F1Callback(Callback):
def __init__(self,
input_key: str = "targets",
output_key: str = "logits",
model_output_key: str = "clipwise_output",
prefix: str = "f1"):
super().__init__(CallbackOrder.Metric)
 self.input_key = input_key
self.output_key = output_key
self.model_output_key = model_output_key
self.prefix = prefix
def on_loader_start(self, state: State):
self.prediction: List[np.ndarray] = []
self.target: List[np.ndarray] = []
def on_batch_end(self, state: State):
targ = state.input[self.input_key].detach().cpu().numpy()
out = state.output[self.output_key]
clipwise_output = out[self.model_output_key].detach().cpu().numpy()
self.prediction.append(clipwise_output)
 self.target.append(targ)
 y_pred = clipwise_output.argmax(axis=1)
 y_true = targ.argmax(axis=1)
 score = f1_score(y_true, y_pred, average="macro")
 state.batch_metrics[self.prefix] = score
def on_loader_end(self, state: State):
y_pred = np.concatenate(self.prediction, axis=0 ).argmax(axis=1)
 y_true = np.concatenate(self.target, axis=0 ).argmax(axis=1)
 score = f1_score(y_true, y_pred, average="macro")
 state.loader_metrics[self.prefix] = score
if state.is_valid_loader:
state.epoch_metrics[state.valid_loader + "_epoch_" +
self.prefix] = score
else:
state.epoch_metrics["train_epoch_" + self.prefix] = score
class mAPCallback(Callback):
def __init__(self,
input_key: str = "targets",
output_key: str = "logits",
model_output_key: str = "clipwise_output",
prefix: str = "mAP"):
super().__init__(CallbackOrder.Metric)
 self.input_key = input_key
self.output_key = output_key
self.model_output_key = model_output_key
self.prefix = prefix
def on_loader_start(self, state: State):
self.prediction: List[np.ndarray] = []
self.target: List[np.ndarray] = []
def on_batch_end(self, state: State):
targ = state.input[self.input_key].detach().cpu().numpy()
out = state.output[self.output_key]
clipwise_output = out[self.model_output_key].detach().cpu().numpy()
self.prediction.append(clipwise_output)
 self.target.append(targ)
 score = average_precision_score(targ, clipwise_output, average=None)
 score = np.nan_to_num(score ).mean()
state.batch_metrics[self.prefix] = score
def on_loader_end(self, state: State):
y_pred = np.concatenate(self.prediction, axis=0)
 y_true = np.concatenate(self.target, axis=0)
 score = average_precision_score(y_true, y_pred, average=None)
 score = np.nan_to_num(score ).mean()
state.loader_metrics[self.prefix] = score
if state.is_valid_loader:
state.epoch_metrics[state.valid_loader + "_epoch_" +
self.prefix] = score
else:
state.epoch_metrics["train_epoch_" + self.prefix] = score<merge>
Id['Target'] = Target
tmp_list = []
for audio_d in TRAIN_RESAMPLED_AUDIO_DIRS:
if not audio_d.exists() :
continue
for ebird_d in audio_d.iterdir() :
if ebird_d.is_file() :
continue
for wav_f in ebird_d.iterdir() :
tmp_list.append([ebird_d.name, wav_f.name, wav_f.as_posix() ])
 train_wav_path_exist = pd.DataFrame(
 tmp_list, columns=["ebird_code", "resampled_filename", "file_path"])
 del tmp_list
train_all = pd.merge(
 train, train_wav_path_exist, on=["ebird_code", "resampled_filename"], how="inner")
 print(train.shape)
 print(train_wav_path_exist.shape)
 print(train_all.shape )<feature_engineering>
submission=Id.copy()
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
 train_all["fold"] = -1
for fold_id,(train_index, val_index)in enumerate(skf.split(train_all, train_all["ebird_code"])) :
train_all.iloc[val_index, -1] = fold_id
fold_proportion = pd.pivot_table(train_all, index="ebird_code", columns="fold", values="xc_id", aggfunc=len)
 print(fold_proportion.shape )<define_variables>
submission.to_csv('submission.csv',index=False )