Kaggle competition: Cornell Birdcall Identification
<SOS> metric: MeanFScoreBeta Kaggle data source: cornell-birdcall-identification<prepare_output>
%matplotlib inline
sns.set_palette(sns.color_palette('tab20', 20))
InteractiveShell.ast_node_interactivity = "all"
init_notebook_mode(connected=True )
X = pd.DataFrame(X,columns=Xcolumns)
 X.head()<define_variables>
nb_start = datetime.now()
plt.rcParams['figure.figsize'] = [16, 10]
plt.rcParams['font.size'] = 14
pd.set_option('display.max_columns', 99)
 pd.set_option('display.max_rows', 99 )
y_log = np.log(y )<split>
@dataclass
class BirdLists:
all = [
'aldfly', 'ameavo', 'amebit', 'amecro', 'amegfi', 'amekes', 'amepip', 'amered', 'amerob', 'amewig', 'amewoo',
'amtspa', 'annhum', 'astfly', 'baisan', 'baleag', 'balori', 'banswa', 'barswa', 'bawwar', 'belkin1', 'belspa2',
'bewwre', 'bkbcuc', 'bkbmag1', 'bkbwar', 'bkcchi', 'bkchum', 'bkhgro', 'bkpwar', 'bktspa', 'blkpho', 'blugrb1',
'blujay', 'bnhcow', 'boboli', 'bongul', 'brdowl', 'brebla', 'brespa', 'brncre', 'brnthr', 'brthum', 'brwhaw',
'btbwar', 'btnwar', 'btywar', 'buffle', 'buggna', 'buhvir', 'bulori', 'bushti', 'buwtea', 'buwwar', 'cacwre',
'calgul', 'calqua', 'camwar', 'cangoo', 'canwar', 'canwre', 'carwre', 'casfin', 'caster1', 'casvir', 'cedwax',
'chispa', 'chiswi', 'chswar', 'chukar', 'clanut', 'cliswa', 'comgol', 'comgra', 'comloo', 'commer', 'comnig',
'comrav', 'comred', 'comter', 'comyel', 'coohaw', 'coshum', 'cowscj1', 'daejun', 'doccor', 'dowwoo', 'dusfly',
'eargre', 'easblu', 'easkin', 'easmea', 'easpho', 'eastow', 'eawpew', 'eucdov', 'eursta', 'evegro', 'fiespa',
'fiscro', 'foxspa', 'gadwal', 'gcrfin', 'gnttow', 'gockin', 'gocspa', 'goleag', 'grbher3', 'grcfly', 'greegr',
'greroa', 'greyel', 'grhowl', 'grnher', 'grtgra', 'grycat', 'gryfly', 'haiwoo', 'hamfly', 'herthr', 'hoomer',
'hoowar', 'horgre', 'horlar', 'houfin', 'houspa', 'houwre', 'indbun', 'juntit1', 'killde', 'labwoo', 'larspa',
'lazbun', 'leabit', 'leafly', 'leasan', 'lecthr', 'lesgol', 'lesnig', 'lesyel', 'lewwoo', 'linspa', 'lobcur',
'lobdow', 'logshr', 'lotduc', 'louwat', 'macwar', 'magwar', 'mallar3', 'marwre', 'merlin', 'moublu', 'mouchi',
'moudov', 'norcar', 'norfli', 'norhar2', 'normoc', 'norpar', 'norpin', 'norsho', 'norwat', 'nrwswa', 'nutwoo',
'olsfly', 'orcwar', 'osprey', 'ovenbi1', 'palwar', 'pasfly', 'pecsan', 'perfal', 'phaino', 'pibgre', 'pilwoo',
'pingro', 'pinjay', 'pinsis', 'pinwar', 'plsvir', 'prawar', 'purfin', 'pygnut', 'rebmer', 'rebnut', 'rebsap',
'rebwoo', 'redcro', 'redhea', 'reevir1', 'renpha', 'reshaw', 'rethaw', 'rewbla', 'ribgul', 'rinduc', 'robgro',
'rocpig', 'rocwre', 'rthhum', 'ruckin', 'rudduc', 'rufgro', 'rufhum', 'rusbla', 'sagspa1', 'sagthr', 'savspa',
'saypho', 'scatan', 'scoori', 'semplo', 'semsan', 'sheowl', 'shshaw', 'snobun', 'snogoo', 'solsan', 'sonspa',
'sora', 'sposan', 'spotow', 'stejay', 'swahaw', 'swaspa', 'swathr', 'treswa', 'truswa', 'tuftit', 'tunswa',
'veery', 'vesspa', 'vigswa', 'warvir', 'wesblu', 'wesgre', 'weskin', 'wesmea', 'wessan', 'westan', 'wewpew',
'whbnut', 'whcspa', 'whfibi', 'whtspa', 'whtswi', 'wilfly', 'wilsni1', 'wiltur', 'winwre3', 'wlswar', 'wooduc',
'wooscj2', 'woothr', 'y00475', 'yebfly', 'yebsap', 'yehbla', 'yelwar', 'yerwar', 'yetvir'
]
top_165 = [
'amecro', 'amegfi', 'amekes', 'amered', 'amerob', 'amewig', 'amtspa', 'annhum', 'baleag', 'balori', 'barswa',
'bawwar', 'belkin1', 'bewwre', 'bkbmag1', 'bkcchi', 'bkhgro', 'blkpho', 'blujay', 'bnhcow', 'bongul', 'brdowl',
'brebla', 'brncre', 'brnthr', 'btbwar', 'btnwar', 'buffle', 'buggna', 'buhvir', 'bushti', 'buwtea', 'calgul',
'calqua', 'cangoo', 'carwre', 'caster1', 'cedwax', 'chispa', 'chiswi', 'chswar', 'cliswa', 'comgol', 'comgra',
'comloo', 'commer', 'comrav', 'comyel', 'coohaw', 'cowscj1', 'daejun', 'doccor', 'dowwoo', 'easblu', 'easkin',
'easmea', 'easpho', 'eastow', 'eawpew', 'eucdov', 'eursta', 'fiespa', 'fiscro', 'foxspa', 'gadwal', 'gockin',
'gocspa', 'grbher3', 'grcfly', 'greegr', 'greyel', 'grhowl', 'grnher', 'grtgra', 'grycat', 'haiwoo', 'herthr',
'hoomer', 'horgre', 'horlar', 'houfin', 'houspa', 'houwre', 'indbun', 'killde', 'leafly', 'leasan', 'lesgol',
'lesyel', 'linspa', 'logshr', 'magwar', 'mallar3', 'marwre', 'merlin', 'mouchi', 'moudov', 'norcar', 'norfli',
'norhar2', 'normoc', 'norpar', 'norpin', 'norsho', 'nrwswa', 'nutwoo', 'orcwar', 'osprey', 'ovenbi1', 'palwar',
'perfal', 'pibgre', 'pilwoo', 'pinsis', 'pinwar', 'purfin', 'rebmer', 'rebnut', 'rebwoo', 'redhea', 'reevir1',
'reshaw', 'rethaw', 'rewbla', 'ribgul', 'rinduc', 'robgro', 'rocpig', 'rthhum', 'ruckin', 'rudduc', 'savspa',
'saypho', 'scatan', 'semplo', 'shshaw', 'snogoo', 'sonspa', 'sposan', 'spotow', 'stejay', 'swaspa', 'swathr',
'treswa', 'tuftit', 'veery', 'vigswa', 'warvir', 'wesblu', 'weskin', 'wesmea', 'westan', 'whbnut', 'whcspa',
'whtspa', 'wilsni1', 'wiltur', 'wlswar', 'wooduc', 'woothr', 'y00475', 'yebsap', 'yelwar', 'yerwar', 'yetvir'
]
non_zero = [
'ameavo', 'amebit', 'amecro', 'amegfi', 'amekes', 'amepip', 'amerob', 'amewig', 'amewoo', 'amtspa', 'annhum',
'astfly', 'baisan', 'baleag', 'banswa', 'belkin1', 'belspa2', 'bkbcuc', 'bkbmag1', 'bkbwar', 'bkcchi', 'bkchum',
'bkpwar', 'bktspa', 'blkpho', 'blugrb1', 'blujay', 'boboli', 'bongul', 'brdowl', 'brebla', 'brespa', 'brthum',
'brwhaw', 'btbwar', 'btnwar', 'btywar', 'buffle', 'buhvir', 'bushti', 'buwtea', 'buwwar', 'cacwre', 'calgul',
'calqua', 'camwar', 'cangoo', 'canwar', 'canwre', 'casfin', 'caster1', 'casvir', 'cedwax', 'chiswi', 'chswar',
'chukar', 'clanut', 'cliswa', 'comgol', 'commer', 'comnig', 'comred', 'comter', 'comyel', 'coohaw', 'coshum',
'cowscj1', 'daejun', 'doccor', 'dusfly', 'eargre', 'easkin', 'easmea', 'eawpew', 'eucdov', 'evegro', 'fiespa',
'fiscro', 'gadwal', 'gcrfin', 'gnttow', 'gocspa', 'goleag', 'grcfly', 'greroa', 'greyel', 'grnher', 'grtgra',
'grycat', 'gryfly', 'hamfly', 'herthr', 'hoomer', 'hoowar', 'horgre', 'horlar', 'indbun', 'juntit1', 'labwoo',
'larspa', 'lazbun', 'leabit', 'leafly', 'leasan', 'lecthr', 'lesgol', 'lesnig', 'lesyel', 'lewwoo', 'linspa',
'lobcur', 'lobdow', 'logshr', 'lotduc', 'louwat', 'macwar', 'magwar', 'merlin', 'moublu', 'mouchi', 'norfli',
'norhar2', 'norpin', 'norsho', 'norwat', 'nrwswa', 'olsfly', 'ovenbi1', 'palwar', 'pasfly', 'pecsan', 'perfal',
'phaino', 'pibgre', 'pingro', 'pinjay', 'pinsis', 'pinwar', 'plsvir', 'prawar', 'purfin', 'pygnut', 'rebmer',
'rebsap', 'rebwoo', 'redhea', 'reevir1', 'renpha', 'reshaw', 'rewbla', 'rinduc', 'robgro', 'rocwre', 'rthhum',
'rudduc', 'rufgro', 'rufhum', 'rusbla', 'sagspa1', 'sagthr', 'saypho', 'scatan', 'scoori', 'semplo', 'semsan',
'sheowl', 'shshaw', 'snobun', 'snogoo', 'solsan', 'sonspa', 'sora', 'sposan', 'stejay', 'swahaw', 'swaspa',
'truswa', 'tunswa', 'veery', 'vesspa', 'vigswa', 'wesblu', 'wesgre', 'weskin', 'wessan', 'westan', 'wewpew',
'whcspa', 'whfibi', 'whtswi', 'wilfly', 'wilsni1', 'wiltur', 'winwre3', 'wooduc', 'wooscj2', 'woothr', 'y00475',
'yebfly', 'yebsap', 'yehbla', 'yetvir'
]
X_train, X_test, y_train, y_test = train_test_split(X, y_log, test_size=0.3, random_state=42 )<normalization>
SAMPLE_RATE = 32000
CHUNK_DURATION = 5
BATCH_SIZE = 64
chunk_len = SAMPLE_RATE * CHUNK_DURATION
xStandardScaler = StandardScaler()
yStandardScaler = StandardScaler()<normalization>
class DropStripes(nn.Module):
def __init__(self, dim, drop_width, stripes_num):

 super(DropStripes, self ).__init__()
assert dim in [2, 3]
self.dim = dim
self.drop_width = drop_width
self.stripes_num = stripes_num
def forward(self, input):

 assert input.ndimension() == 4
if self.training is False:
return input
else:
batch_size = input.shape[0]
total_width = input.shape[self.dim]
for n in range(batch_size):
self.transform_slice(input[n], total_width)
 return input
def transform_slice(self, e, total_width):

 for _ in range(self.stripes_num):
distance = torch.randint(low=0, high=self.drop_width, size=(1,)) [0]
bgn = torch.randint(low=0, high=total_width - distance, size=(1,)) [0]
if self.dim == 2:
e[:, bgn : bgn + distance, :] = 0
elif self.dim == 3:
e[:, :, bgn : bgn + distance] = 0
class SpecAugmentation(nn.Module):
def __init__(self, time_drop_width, time_stripes_num, freq_drop_width,
freq_stripes_num):

 super(SpecAugmentation, self ).__init__()
self.time_dropper = DropStripes(dim=2, drop_width=time_drop_width,
stripes_num=time_stripes_num)
 self.freq_dropper = DropStripes(dim=3, drop_width=freq_drop_width,
stripes_num=freq_stripes_num)
 def forward(self, input):
x = self.time_dropper(input)
 x = self.freq_dropper(x)
 return x
X_train = xStandardScaler.fit_transform(X_train)
 X_test = xStandardScaler.transform(X_test )<normalization>
def do_mixup(x, mixup_lambda):

 out =(x[0:: 2].transpose(0, -1)* mixup_lambda[0:: 2] + \
x[1:: 2].transpose(0, -1)* mixup_lambda[1:: 2] ).transpose(0, -1)
 return out
def interpolate(x, ratio):

(batch_size, time_steps, classes_num)= x.shape
upsampled = x[:, :, None, :].repeat(1, 1, ratio, 1)
 upsampled = upsampled.reshape(batch_size, time_steps * ratio, classes_num)
 return upsampled
def pad_framewise_output(framewise_output, frames_num):

 pad = framewise_output[:, -1:, :].repeat(1, frames_num - framewise_output.shape[1], 1)

output = torch.cat(( framewise_output, pad), dim=1)

return output
def init_layer(layer):

 nn.init.xavier_uniform_(layer.weight)
 if hasattr(layer, 'bias'):
if layer.bias is not None:
layer.bias.data.fill_(0.)
def init_bn(bn):

 bn.bias.data.fill_(0.)
bn.weight.data.fill_(1.)
class ConvBlock(nn.Module):
def __init__(self, in_channels, out_channels):
super(ConvBlock, self ).__init__()
self.conv1 = nn.Conv2d(in_channels=in_channels,
out_channels=out_channels,
kernel_size=(3, 3), stride=(1, 1),
padding=(1, 1), bias=False)
 self.conv2 = nn.Conv2d(in_channels=out_channels,
out_channels=out_channels,
kernel_size=(3, 3), stride=(1, 1),
padding=(1, 1), bias=False)
 self.bn1 = nn.BatchNorm2d(out_channels)
 self.bn2 = nn.BatchNorm2d(out_channels)
 self.init_weight()
def init_weight(self):
init_layer(self.conv1)
 init_layer(self.conv2)
 init_bn(self.bn1)
 init_bn(self.bn2)
 def forward(self, input, pool_size=(2, 2), pool_type='avg'):
x = input
x = F.relu_(self.bn1(self.conv1(x)))
 x = F.relu_(self.bn2(self.conv2(x)))
 if pool_type == 'max':
x = F.max_pool2d(x, kernel_size=pool_size)
 elif pool_type == 'avg':
x = F.avg_pool2d(x, kernel_size=pool_size)
 elif pool_type == 'avg+max':
x1 = F.avg_pool2d(x, kernel_size=pool_size)
 x2 = F.max_pool2d(x, kernel_size=pool_size)
 x = x1 + x2
else:
raise Exception('Incorrect argument!')
 return x
class ConvBlock5x5(nn.Module):
def __init__(self, in_channels, out_channels):
super(ConvBlock5x5, self ).__init__()
self.conv1 = nn.Conv2d(in_channels=in_channels,
out_channels=out_channels,
kernel_size=(5, 5), stride=(1, 1),
padding=(2, 2), bias=False)
 self.bn1 = nn.BatchNorm2d(out_channels)
 self.init_weight()
def init_weight(self):
init_layer(self.conv1)
 init_bn(self.bn1)
 def forward(self, input, pool_size=(2, 2), pool_type='avg'):
x = input
x = F.relu_(self.bn1(self.conv1(x)))
 if pool_type == 'max':
x = F.max_pool2d(x, kernel_size=pool_size)
 elif pool_type == 'avg':
x = F.avg_pool2d(x, kernel_size=pool_size)
 elif pool_type == 'avg+max':
x1 = F.avg_pool2d(x, kernel_size=pool_size)
 x2 = F.max_pool2d(x, kernel_size=pool_size)
 x = x1 + x2
else:
raise Exception('Incorrect argument!')
 return x
class AttBlock(nn.Module):
def __init__(self, n_in, n_out, activation='linear', temperature=1.) :
super(AttBlock, self ).__init__()
self.activation = activation
self.temperature = temperature
self.att = nn.Conv1d(in_channels=n_in, out_channels=n_out, kernel_size=1, stride=1, padding=0, bias=True)
 self.cla = nn.Conv1d(in_channels=n_in, out_channels=n_out, kernel_size=1, stride=1, padding=0, bias=True)
 self.bn_att = nn.BatchNorm1d(n_out)
 self.init_weights()
def init_weights(self):
init_layer(self.att)
 init_layer(self.cla)
 init_bn(self.bn_att)
 def forward(self, x):
norm_att = torch.softmax(torch.clamp(self.att(x), -10, 10), dim=-1)
 cla = self.nonlinear_transform(self.cla(x))
x = torch.sum(norm_att * cla, dim=2)
 return x, norm_att, cla
def nonlinear_transform(self, x):
if self.activation == 'linear':
return x
elif self.activation == 'sigmoid':
return torch.sigmoid(x)
 class Cnn14(nn.Module):
def __init__(self, sample_rate, window_size, hop_size, mel_bins, fmin,
fmax, classes_num):
super(Cnn14, self ).__init__()
window = 'hann'
center = True
pad_mode = 'reflect'
ref = 1.0
amin = 1e-10
top_db = None
self.dataset_mean = 0.
 self.dataset_std = 1.
 self.spectrogram_extractor = Spectrogram(n_fft=window_size, hop_length=hop_size,
win_length=window_size, window=window, center=center,
pad_mode=pad_mode,
freeze_parameters=True)
 self.logmel_extractor = LogmelFilterBank(sr=sample_rate, n_fft=window_size,
n_mels=mel_bins, fmin=fmin, fmax=fmax, ref=ref, amin=amin,
top_db=top_db,
freeze_parameters=True)
 self.spec_augmenter = SpecAugmentation(time_drop_width=32, time_stripes_num=2,
freq_drop_width=8, freq_stripes_num=2)
 self.bn0 = nn.BatchNorm2d(64)
 self.conv_block1 = ConvBlock(in_channels=1, out_channels=64)
 self.conv_block2 = ConvBlock(in_channels=64, out_channels=128)
 self.conv_block3 = ConvBlock(in_channels=128, out_channels=256)
 self.conv_block4 = ConvBlock(in_channels=256, out_channels=512)
 self.conv_block5 = ConvBlock(in_channels=512, out_channels=1024)
 self.conv_block6 = ConvBlock(in_channels=1024, out_channels=2048)
 self.fc1 = nn.Linear(2048, 2048, bias=True)
 self.fc_audioset = nn.Linear(2048, classes_num, bias=True)
 self.init_weight()
def init_weight(self):
init_bn(self.bn0)
 init_layer(self.fc1)
 init_layer(self.fc_audioset)
 def forward(self, input, mixup_lambda=None):

 x = self.spectrogram_extractor(input)
 x = self.logmel_extractor(x)
 x =(x - self.dataset_mean)/ self.dataset_std
x = x.transpose(1, 3)
 x = self.bn0(x)
 x = x.transpose(1, 3)
 if self.training:
x = self.spec_augmenter(x)
 if self.training and mixup_lambda is not None:
x = do_mixup(x, mixup_lambda)
 x = self.conv_block1(x, pool_size=(2, 2), pool_type='avg')
 x = F.dropout(x, p=0.2, training=self.training)
 x = self.conv_block2(x, pool_size=(2, 2), pool_type='avg')
 x = F.dropout(x, p=0.2, training=self.training)
 x = self.conv_block3(x, pool_size=(2, 2), pool_type='avg')
 x = F.dropout(x, p=0.2, training=self.training)
 x = self.conv_block4(x, pool_size=(2, 2), pool_type='avg')
 x = F.dropout(x, p=0.2, training=self.training)
 x = self.conv_block5(x, pool_size=(2, 2), pool_type='avg')
 x = F.dropout(x, p=0.2, training=self.training)
 x = self.conv_block6(x, pool_size=(1, 1), pool_type='avg')
 x = F.dropout(x, p=0.2, training=self.training)
 x = torch.mean(x, dim=3)
(x1, _)= torch.max(x, dim=2)
 x2 = torch.mean(x, dim=2)
 x = x1 + x2
x = F.dropout(x, p=0.5, training=self.training)
 x = F.relu_(self.fc1(x))
embedding = F.dropout(x, p=0.5, training=self.training)
 clipwise_output = torch.sigmoid(self.fc_audioset(x))
output_dict = {'clipwise_output': clipwise_output, 'embedding': embedding}
return output_dict
y_train = yStandardScaler.fit_transform(y_train.ravel().reshape(-1, 1))
y_test = yStandardScaler.transform(y_test.ravel().reshape(-1, 1))<define_variables>
def _resnet_conv3x3(in_planes, out_planes):
return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=1,
padding=1, groups=1, bias=False, dilation=1)
 def _resnet_conv1x1(in_planes, out_planes):
return nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=1, bias=False)
 class _ResnetBasicBlock(nn.Module):
expansion = 1
def __init__(self, inplanes, planes, stride=1, downsample=None, groups=1,
base_width=64, dilation=1, norm_layer=None):
super(_ResnetBasicBlock, self ).__init__()
if norm_layer is None:
norm_layer = nn.BatchNorm2d
if groups != 1 or base_width != 64:
raise ValueError('_ResnetBasicBlock only supports groups=1 and base_width=64')
 if dilation > 1:
raise NotImplementedError("Dilation > 1 not supported in _ResnetBasicBlock")
 self.stride = stride
self.conv1 = _resnet_conv3x3(inplanes, planes)
 self.bn1 = norm_layer(planes)
 self.relu = nn.ReLU(inplace=True)
 self.conv2 = _resnet_conv3x3(planes, planes)
 self.bn2 = norm_layer(planes)
 self.downsample = downsample
self.stride = stride
self.init_weights()
def init_weights(self):
init_layer(self.conv1)
 init_bn(self.bn1)
 init_layer(self.conv2)
 init_bn(self.bn2)
 nn.init.constant_(self.bn2.weight, 0)
 def forward(self, x):
identity = x
if self.stride == 2:
out = F.avg_pool2d(x, kernel_size=(2, 2))
else:
out = x
out = self.conv1(out)
 out = self.bn1(out)
 out = self.relu(out)
 out = F.dropout(out, p=0.1, training=self.training)
 out = self.conv2(out)
 out = self.bn2(out)
 if self.downsample is not None:
identity = self.downsample(identity)
 out += identity
out = self.relu(out)
 return out
class _ResnetBottleneck(nn.Module):
expansion = 4
def __init__(self, inplanes, planes, stride=1, downsample=None, groups=1,
base_width=64, dilation=1, norm_layer=None):
super(_ResnetBottleneck, self ).__init__()
if norm_layer is None:
norm_layer = nn.BatchNorm2d
width = int(planes *(base_width / 64.))* groups
self.stride = stride
self.conv1 = _resnet_conv1x1(inplanes, width)
 self.bn1 = norm_layer(width)
 self.conv2 = _resnet_conv3x3(width, width)
 self.bn2 = norm_layer(width)
 self.conv3 = _resnet_conv1x1(width, planes * self.expansion)
 self.bn3 = norm_layer(planes * self.expansion)
 self.relu = nn.ReLU(inplace=True)
 self.downsample = downsample
self.stride = stride
self.init_weights()
def init_weights(self):
init_layer(self.conv1)
 init_bn(self.bn1)
 init_layer(self.conv2)
 init_bn(self.bn2)
 init_layer(self.conv3)
 init_bn(self.bn3)
 nn.init.constant_(self.bn3.weight, 0)
 def forward(self, x):
identity = x
if self.stride == 2:
x = F.avg_pool2d(x, kernel_size=(2, 2))
out = self.conv1(x)
 out = self.bn1(out)
 out = self.relu(out)
 out = self.conv2(out)
 out = self.bn2(out)
 out = self.relu(out)
 out = F.dropout(out, p=0.1, training=self.training)
 out = self.conv3(out)
 out = self.bn3(out)
 if self.downsample is not None:
identity = self.downsample(identity)
 out += identity
out = self.relu(out)
 return out
class _ResNet(nn.Module):
def __init__(self, block, layers, zero_init_residual=False,
groups=1, width_per_group=64, replace_stride_with_dilation=None,
norm_layer=None):
super(_ResNet, self ).__init__()
if norm_layer is None:
norm_layer = nn.BatchNorm2d
self._norm_layer = norm_layer
self.inplanes = 64
self.dilation = 1
if replace_stride_with_dilation is None:
replace_stride_with_dilation = [False, False, False]
if len(replace_stride_with_dilation)!= 3:
raise ValueError("replace_stride_with_dilation should be None "
"or a 3-element tuple, got {}".format(replace_stride_with_dilation))
self.groups = groups
self.base_width = width_per_group
self.layer1 = self._make_layer(block, 64, layers[0], stride=1)
 self.layer2 = self._make_layer(block, 128, layers[1], stride=2,
dilate=replace_stride_with_dilation[0])
 self.layer3 = self._make_layer(block, 256, layers[2], stride=2,
dilate=replace_stride_with_dilation[1])
 self.layer4 = self._make_layer(block, 512, layers[3], stride=2,
dilate=replace_stride_with_dilation[2])
 def _make_layer(self, block, planes, blocks, stride=1, dilate=False):
norm_layer = self._norm_layer
downsample = None
previous_dilation = self.dilation
if dilate:
self.dilation *= stride
stride = 1
if stride != 1 or self.inplanes != planes * block.expansion:
if stride == 1:
downsample = nn.Sequential(
 _resnet_conv1x1(self.inplanes, planes * block.expansion),
norm_layer(planes * block.expansion), 
)
 init_layer(downsample[0])
 init_bn(downsample[1])
 elif stride == 2:
downsample = nn.Sequential(
 nn.AvgPool2d(kernel_size=2),
_resnet_conv1x1(self.inplanes, planes * block.expansion),
norm_layer(planes * block.expansion), 
)
 init_layer(downsample[1])
 init_bn(downsample[2])
 layers = []
layers.append(block(self.inplanes, planes, stride, downsample, self.groups,
self.base_width, previous_dilation, norm_layer))
self.inplanes = planes * block.expansion
for _ in range(1, blocks):
layers.append(block(self.inplanes, planes, groups=self.groups,
base_width=self.base_width, dilation=self.dilation,
norm_layer=norm_layer))
return nn.Sequential(*layers)
 def forward(self, x):
x = self.layer1(x)
 x = self.layer2(x)
 x = self.layer3(x)
 x = self.layer4(x)
 return x
class ResNet38(nn.Module):
def __init__(self, sample_rate, window_size, hop_size, mel_bins, fmin,
fmax, classes_num):
super(ResNet38, self ).__init__()
window = 'hann'
center = True
pad_mode = 'reflect'
ref = 1.0
amin = 1e-10
top_db = None
self.spectrogram_extractor = Spectrogram(n_fft=window_size, hop_length=hop_size,
win_length=window_size, window=window, center=center,
pad_mode=pad_mode,
freeze_parameters=True)
 self.logmel_extractor = LogmelFilterBank(sr=sample_rate, n_fft=window_size,
n_mels=mel_bins, fmin=fmin, fmax=fmax, ref=ref, amin=amin,
top_db=top_db,
freeze_parameters=True)
 self.spec_augmenter = SpecAugmentation(time_drop_width=64, time_stripes_num=2,
freq_drop_width=8, freq_stripes_num=2)
 self.bn0 = nn.BatchNorm2d(64)
 self.conv_block1 = ConvBlock(in_channels=1, out_channels=64)
 self.resnet = _ResNet(block=_ResnetBasicBlock, layers=[3, 4, 6, 3], zero_init_residual=True)
 self.conv_block_after1 = ConvBlock(in_channels=512, out_channels=2048)
 self.fc1 = nn.Linear(2048, 2048)
 self.fc_audioset = nn.Linear(2048, classes_num, bias=True)
 self.init_weights()
def init_weights(self):
init_bn(self.bn0)
 init_layer(self.fc1)
 init_layer(self.fc_audioset)
 def forward(self, input, mixup_lambda=None):

 x = self.spectrogram_extractor(input)
 x = self.logmel_extractor(x)
 x = x.transpose(1, 3)
 x = self.bn0(x)
 x = x.transpose(1, 3)
 if self.training:
x = self.spec_augmenter(x)
 if self.training and mixup_lambda is not None:
x = do_mixup(x, mixup_lambda)
 x = self.conv_block1(x, pool_size=(2, 2), pool_type='avg')
 x = F.dropout(x, p=0.2, training=self.training, inplace=True)
 x = self.resnet(x)
 x = F.avg_pool2d(x, kernel_size=(2, 2))
x = F.dropout(x, p=0.2, training=self.training, inplace=True)
 x = self.conv_block_after1(x, pool_size=(1, 1), pool_type='avg')
 x = F.dropout(x, p=0.2, training=self.training, inplace=True)
 x = torch.mean(x, dim=3)
(x1, _)= torch.max(x, dim=2)
 x2 = torch.mean(x, dim=2)
 x = x1 + x2
x = F.dropout(x, p=0.5, training=self.training)
 x = F.relu_(self.fc1(x))
embedding = F.dropout(x, p=0.5, training=self.training)
 clipwise_output = torch.sigmoid(self.fc_audioset(x))
output_dict = {'clipwise_output': clipwise_output, 'embedding': embedding}
return output_dict
ModelCompList = list()<train_on_grid>
_window = 'hann'
_center = True
_pad_mode = 'reflect'
_ref = 1.0
_amin = 1e-10
_top_db = None
_model_config = {
"sample_rate": 32000,
"window_size": 1024,
"hop_size": 320,
"mel_bins": 64,
"fmin": 50,
"fmax": 14000,
"classes_num": 527
}
def my_cnn14(n_fft, n_mels, n_classes=100, hop_size=320, fmin=160, fmax=10300):
model = Cnn14(**_model_config)
 model.fc_audioset = nn.Linear(2048, n_classes, bias=True)
 init_layer(model.fc_audioset)
 model.spectrogram_extractor = Spectrogram(
 n_fft=n_fft, hop_length=hop_size, win_length=n_fft, window=_window,
center=_center, pad_mode=_pad_mode, freeze_parameters=True)
 model.logmel_extractor = LogmelFilterBank(
 sr=SAMPLE_RATE, n_fft=n_fft, n_mels=n_mels, fmin=fmin, fmax=fmax,
ref=_ref, amin=_amin, top_db=_top_db, freeze_parameters=True)
 model.bn0 = nn.BatchNorm2d(n_mels)
 init_bn(model.bn0)
 return model
alphas = np.linspace(0,.1,num=21)
 lgLasso = LassoCV(cv=10,alphas=alphas)
 lgLasso.fit(X_train,y_train )<normalization>
def read_audio_fast(path):
clip, sr_native = librosa.core.audio.__audioread_load(
 path, offset=0.0, duration=None, dtype=np.float32)
 clip = librosa.to_mono(clip)
 if sr_native > 0:
clip = librosa.resample(clip, sr_native, SAMPLE_RATE, res_type='kaiser_fast')
 return clip, SAMPLE_RATE
def random_chunk(y, duration, sr=SAMPLE_RATE):
sample_size = int(duration * sr)
 if y.shape[0] < sample_size:
raise ValueError('Too short clip')
 t = np.random.randint(0, y.shape[0] - sample_size)
 return y[t: t + sample_size]
print(lgLasso.alpha_ )<predict_on_test>
if os.path.exists('/kaggle/input/birdsong-recognition/test_audio'):
BASE_TEST_DIR = '/kaggle/input/birdsong-recognition'
TEST_AUDIO_DIR = '/kaggle/input/birdsong-recognition/test_audio'
else:
BASE_TEST_DIR = '/kaggle/input/cornellfaketest'
TEST_AUDIO_DIR = '/kaggle/input/cornellfaketest'
print(BASE_TEST_DIR, TEST_AUDIO_DIR)
 df_test = pd.read_csv(f'{BASE_TEST_DIR}/test.csv')
 df_test.shape
df_test.head()
df_test.groupby('site' ).nunique()
predictionsLinear = lgLasso.predict(X_test )<normalization>
audio_site = {a: s for a, s in df_test[['audio_id', 'site']].drop_duplicates().values}
plt.scatter(np.exp(yStandardScaler.inverse_transform(y_test)) ,np.exp(yStandardScaler.inverse_transform(predictionsLinear)) )<normalization>
def get_model_predictions_for_clip_fast(y, model, device, classes):
duration = y.shape[0] // SAMPLE_RATE
batch = []
start_seconds = []
for start in range(0, duration - CHUNK_DURATION + 1, 5):
end = start + 5
start_seconds.append(start)
 chunk = y[start * SAMPLE_RATE: end * SAMPLE_RATE]
if len(chunk)!= CHUNK_DURATION * SAMPLE_RATE:
print(chunk.shape)
 break
batch.append(chunk)
 batch = np.asarray(batch)
 tensors = torch.from_numpy(batch)
 tensors = tensors.to(device)
 with torch.no_grad() :
preds = model(tensors)['clipwise_output']
test_preds = preds.cpu().numpy()
pred_df = pd.DataFrame(test_preds, columns=classes)
 pred_df['start_second'] = start_seconds
return pred_df
sns.distplot(( yStandardScaler.inverse_transform(y_test)-yStandardScaler.inverse_transform(predictionsLinear)) ,bins=50);<compute_test_metric>
sites = ['site_1', 'site_2', 'site_3']
audio_ids_by_site = {}
for s in sites:
audio_ids_by_site[s] = list(df_test[df_test.site == s].audio_id.unique())
 print(s, len(audio_ids_by_site[s]))
def evaluateModel(name,y_test,Predictions):
modelComp = dict()
modelComp['Name'] = name
modelComp['MAE'] = metrics.mean_absolute_error(np.exp(yStandardScaler.inverse_transform(y_test)) ,
np.exp(yStandardScaler.inverse_transform(Predictions)))
 modelComp['MSE'] = metrics.mean_squared_error(np.exp(yStandardScaler.inverse_transform(y_test)) ,
np.exp(yStandardScaler.inverse_transform(Predictions)))
 modelComp['RMSE'] = np.sqrt(metrics.mean_squared_error(np.exp(yStandardScaler.inverse_transform(y_test)) ,
np.exp(yStandardScaler.inverse_transform(Predictions))))
print('MAE:', modelComp['MAE'])
 print('MSE:', modelComp['MSE'])
 print('RMSE:', modelComp['RMSE'])
 ModelCompList.append(modelComp )<compute_test_metric>
models = [ 
(
 'models_CNN14_CV1_C262_M1_N1_7045_14005959.pth',
{'n_fft': 1536, 'hop_size': 320, 'n_mels': 128, 'n_classes': 262, 'fmin': 300, 'fmax': 11000} 
), 
(
 'models_CNN14_CV3_C165_M3_N1_F18256_14102207.pth',
{'n_fft': 1536, 'hop_size': 320, 'n_mels': 128, 'n_classes': 165, 'fmin': 300, 'fmax': 11000} 
), 
(
 'models_CNN14_CV3_C262_M0_N1_7063_14141546.pth',
{'n_fft': 1536, 'hop_size': 320, 'n_mels': 128, 'n_classes': 262, 'fmin': 300, 'fmax': 11000} 
)
 ]
evaluateModel('Lasso',y_test,predictionsLinear )<train_on_grid>
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 def load_model(model_name, model_config):
model = my_cnn14(**model_config)
 model.load_state_dict(torch.load(f'/kaggle/input/cornellmodels/{model_name}', map_location=device))
return model
alphas = np.linspace(0,20,num=21)
 lgRidge = RidgeCV(cv=10,alphas=alphas)
 lgRidge.fit(X_train,y_train )<normalization>
def get_spec_stats(ys, model):
test_means = []
test_stds = []
for y in tqdm(ys):
duration = y.shape[0] // SAMPLE_RATE
batch = []
for start in range(0, duration, 5):
end = start + 5
chunk = y[start * SAMPLE_RATE: end * SAMPLE_RATE]
batch.append(chunk)
 batch = np.array(batch)
 batch = torch.from_numpy(batch)
 tensors = batch.to(device)
 specs = model.spectrogram_extractor(tensors)
 specs = model.logmel_extractor(specs)
 specs = specs.cpu().numpy()
for spec in specs:
test_means.append(spec.mean())
 test_stds.append(spec.std())
 test_mean = np.mean(test_means)
 test_std = np.mean(test_stds)
 print(f'Test stats: {test_mean:.3} +- {test_std:.3}')
 return test_mean, test_std
print(lgRidge.alpha_ )<predict_on_test>
site12_preds = []
for site in ['site_1', 'site_2']:
test_audios = {}
for audio_id in tqdm(audio_ids_by_site[site]):
path = os.path.join(TEST_AUDIO_DIR, f'{audio_id}.mp3')
 y, _ = read_audio_fast(path)
 test_audios[audio_id] = y
for model_name, model_config in models:
model = load_model(model_name, model_config)
 model = model.to(device)
 _ = model.eval()
if '_C262_' in model_name:
classes = BirdLists.all
elif '_C165_' in model_name:
classes = BirdLists.top_165
elif '_C202_' in model_name:
classes = BirdLists.non_zero
if '_N1_' in model_name:
test_mean, test_std = get_spec_stats(test_audios.values() , model)
 model.dataset_mean = test_mean
model.dataset_std = test_std
for audio_id, y in tqdm(test_audios.items()):
pred_df = get_model_predictions_for_clip_fast(y, model, device, classes)
 pred_df['audio_id'] = audio_id
pred_df['site'] = site
site12_preds.append(pred_df)
 predictions_12 = pd.concat(site12_preds )
predictionsLinear = lgRidge.predict(X_test )<normalization>
zero = [
'redcro', 'norcar', 'houwre', 'spotow', 'bewwre', 'carwre', 'swathr', 'comrav', 'warvir', 'whbnut', 'marwre',
'normoc', 'wesmea', 'amered', 'whtspa', 'houfin', 'ruckin', 'eastow', 'tuftit', 'chispa', 'grhowl', 'savspa',
'wlswar', 'buggna', 'brncre', 'aldfly', 'orcwar', 'bkhgro', 'bnhcow', 'houspa', 'yerwar', 'haiwoo', 'dowwoo',
'bawwar', 'foxspa', 'balori', 'rebnut', 'eursta', 'gockin', 'killde', 'comgra', 'barswa', 'osprey', 'mallar3',
'pilwoo', 'moudov', 'easpho', 'comloo', 'treswa', 'easblu', 'rethaw', 'grbher3', 'yelwar', 'greegr', 'ribgul',
'nutwoo', 'rocpig', 'brnthr', 'norpar', 'bulori',
]
common = ['whcspa', 'grycat', 'herthr', 'amerob', 'reevir1', 'comyel', 'norfli', 'amecro']
rare = [
'doccor', 'baleag', 'buffle', 'amekes', 'annhum', 'reshaw', 'coohaw', 'lesgol', 'norsho', 'commer', 'norhar2',
'rinduc', 'amewig', 'blkpho', 'hoomer', 'comgol', 'cowscj1', 'wiltur', 'buwtea', 'rudduc', 'bushti', 'rebmer',
'stejay', 'norpin', 'vigswa', 'gocspa', 'redhea', 'wesblu', 'shshaw', 'horgre', 'calqua', 'perfal', 'labwoo',
'wewpew', 'bkchum', 'truswa', 'lotduc', 'brwhaw', 'pasfly', 'swahaw', 'eargre', 'whfibi', 'tunswa', 'phaino',
'brthum', 'rufgro', 'wooscj2', 'cacwre', 'rebsap', 'whtswi', 'goleag', 'snobun', 'renpha', 'coshum', 'baisan',
'sheowl', 'lewwoo', 'lesnig', 'gcrfin', 'sagspa1', 'chukar', 'lecthr',
]
BIRD_THRESHOLD = 0.33
THRESHOLDS = {c: BIRD_THRESHOLD for c in BirdLists.all}
for c in zero:
THRESHOLDS[c] = 0.6
for c in common:
THRESHOLDS[c] = 0.3
for c in rare:
THRESHOLDS[c] = 0.55
THRESHOLDS
plt.scatter(np.exp(yStandardScaler.inverse_transform(y_test)) ,np.exp(yStandardScaler.inverse_transform(predictionsLinear)) )<normalization>
mean_predictions_12 = predictions_12.fillna(0 ).groupby(['site', 'audio_id', 'start_second'] ).mean().reset_index()
mean_predictions_12['birdmax'] = mean_predictions_12[BirdLists.all].max(axis=1)
 mean_predictions_12['seconds'] = mean_predictions_12.start_second + CHUNK_DURATION
mean_predictions_12['row_id'] = mean_predictions_12.site + '_' + mean_predictions_12.audio_id + '_' + mean_predictions_12.seconds.astype(str)
 multi_birds = []
for _, row in mean_predictions_12[BirdLists.all].iterrows() :
guesses = ' '.join([bird for bird, p in row.items() if p >= THRESHOLDS[bird]])
 multi_birds.append(guesses)
 mean_predictions_12['bird_guesses'] = multi_birds
mean_predictions_12.shape
mean_predictions_12.head()
sns.distplot(( yStandardScaler.inverse_transform(y_test)-yStandardScaler.inverse_transform(predictionsLinear)) ,bins=50);<compute_test_metric>
site3_preds = []
site = 'site_3'
test_audios = {}
for audio_id in tqdm(audio_ids_by_site[site]):
path = os.path.join(TEST_AUDIO_DIR, f'{audio_id}.mp3')
 y, _ = read_audio_fast(path)
 test_audios[audio_id] = y
for model_name, model_config in models[:1]:
model = load_model(model_name, model_config)
 model = model.to(device)
 _ = model.eval()
classes = BirdLists.all
for audio_id, y in tqdm(test_audios.items()):
pred_df = get_model_predictions_for_clip_fast(y, model, device, classes)
 pred_df['audio_id'] = audio_id
pred_df['site'] = site
site3_preds.append(pred_df)
 predictions_3 = pd.concat(site3_preds )
evaluateModel('Ridge',y_test,predictionsLinear )<train_on_grid>
predictions_3['birdmax'] = predictions_3[BirdLists.all].max(axis=1)
 predictions_3['bird_guess'] = [BirdLists.all[idx] for idx in predictions_3[BirdLists.all].values.argmax(axis=1)]
alphas = np.linspace(0,10,num=21)
 l1_ratio = np.linspace(0,1,num=21)
 lgNet = ElasticNetCV(cv=10,alphas=alphas,l1_ratio=l1_ratio)
 lgNet.fit(X_train,y_train )<normalization>
guesses_3 = predictions_3[predictions_3.birdmax > 0.5].groupby(['audio_id', 'bird_guess'])[['site']].count().reset_index()
guesses_3
print(lgNet.alpha_)
 print(lgNet.l1_ratio_ )<predict_on_test>
site_3_birds = []
for audio_id, df in guesses_3.groupby('audio_id'):
site_3_birds.append([audio_id, ' '.join(df.bird_guess.values)])
 site_3_birds = pd.DataFrame(site_3_birds, columns=['audio_id', 'bird_guesses'])
 site_3_birds = site_3_birds.merge(df_test[['audio_id', 'row_id']], how='left', on='audio_id')
 site_3_birds
predictionsLinear = lgNet.predict(X_test )<normalization>
guesses = pd.concat([
site12_birds[['row_id', 'bird_guesses']],
site_3_birds[['row_id', 'bird_guesses']]
])
 guesses.shape
guesses.head()
guesses.tail()
plt.scatter(np.exp(yStandardScaler.inverse_transform(y_test)) ,np.exp(yStandardScaler.inverse_transform(predictionsLinear)) )<normalization>
submission = pd.read_csv('/kaggle/input/birdsong-recognition/sample_submission.csv')
 submission.head()
submission.shape
submission = submission.merge(guesses[['row_id', 'bird_guesses']], how='left', on=['row_id'])
 idx =(submission.bird_guesses.str.len() > 1)
 submission.loc[idx, 'birds'] = submission.loc[idx, 'bird_guesses']
submission
sns.distplot(( yStandardScaler.inverse_transform(y_test)-yStandardScaler.inverse_transform(predictionsLinear)) ,bins=50);<compute_test_metric>
submission[['row_id', 'birds']].to_csv('submission.csv', index=False )
<choose_model_class><EOS> 
end = datetime.now()
print(end,(end - nb_start ).seconds )