Kaggle competition: Cornell Birdcall Identification
<SOS> metric: MeanFScoreBeta Kaggle data source: cornell-birdcall-identification<split>
!pip install.. /input/geffnet-pack/gen-efficientnet-pytorch-master/ > /dev/null
!pip install.. /input/resnest-git/ResNeSt-master/ > /dev/null
train_cluster_pca = train_cluster_pca.iloc[:,650:]
test_cluster_pca = test_cluster_pca.iloc[:,650:]<split>
import math
from glob import glob
from os.path import join as pjoin
from tqdm.notebook import tqdm
from typing import Mapping, Any, List, Tuple, Optional, Union
from pathlib import Path
import torch
import torch.nn as nn
import torch.nn.functional as F
import numpy as np
import pandas as pd
import librosa
import geffnet
from resnest.torch import resnest50_fast_1s1x64d
from torchvision.models.resnet import ResNet, Bottleneck
train_features_cluster=train_features2.iloc[:,876:]
test_features_cluster=test_features2.iloc[:,876:]<define_variables>
TARGET_SR=32_000
PP_NORMALIZE=True
MIN_SEC = 1
DEVICE = 'cuda'
BATCH_SIZE = 64
SIGMOID_THRESH = 0.3
MAX_BIRDS = None
gsquarecols=['g-574','g-211','g-216','g-0','g-255','g-577','g-153','g-389','g-60','g-370','g-248','g-167','g-203','g-177','g-301','g-332','g-517','g-6','g-744','g-224','g-162','g-3','g-736','g-486','g-283','g-22','g-359','g-361','g-440','g-335','g-106','g-307','g-745','g-146','g-416','g-298','g-666','g-91','g-17','g-549','g-145','g-157','g-768','g-568','g-396']<feature_engineering>
BIRD_CODE = {
'aldfly': 0, 'ameavo': 1, 'amebit': 2, 'amecro': 3, 'amegfi': 4,
'amekes': 5, 'amepip': 6, 'amered': 7, 'amerob': 8, 'amewig': 9,
'amewoo': 10, 'amtspa': 11, 'annhum': 12, 'astfly': 13, 'baisan': 14,
'baleag': 15, 'balori': 16, 'banswa': 17, 'barswa': 18, 'bawwar': 19,
'belkin1': 20, 'belspa2': 21, 'bewwre': 22, 'bkbcuc': 23, 'bkbmag1': 24,
'bkbwar': 25, 'bkcchi': 26, 'bkchum': 27, 'bkhgro': 28, 'bkpwar': 29,
'bktspa': 30, 'blkpho': 31, 'blugrb1': 32, 'blujay': 33, 'bnhcow': 34,
'boboli': 35, 'bongul': 36, 'brdowl': 37, 'brebla': 38, 'brespa': 39,
'brncre': 40, 'brnthr': 41, 'brthum': 42, 'brwhaw': 43, 'btbwar': 44,
'btnwar': 45, 'btywar': 46, 'buffle': 47, 'buggna': 48, 'buhvir': 49,
'bulori': 50, 'bushti': 51, 'buwtea': 52, 'buwwar': 53, 'cacwre': 54,
'calgul': 55, 'calqua': 56, 'camwar': 57, 'cangoo': 58, 'canwar': 59,
'canwre': 60, 'carwre': 61, 'casfin': 62, 'caster1': 63, 'casvir': 64,
'cedwax': 65, 'chispa': 66, 'chiswi': 67, 'chswar': 68, 'chukar': 69,
'clanut': 70, 'cliswa': 71, 'comgol': 72, 'comgra': 73, 'comloo': 74,
'commer': 75, 'comnig': 76, 'comrav': 77, 'comred': 78, 'comter': 79,
'comyel': 80, 'coohaw': 81, 'coshum': 82, 'cowscj1': 83, 'daejun': 84,
'doccor': 85, 'dowwoo': 86, 'dusfly': 87, 'eargre': 88, 'easblu': 89,
'easkin': 90, 'easmea': 91, 'easpho': 92, 'eastow': 93, 'eawpew': 94,
'eucdov': 95, 'eursta': 96, 'evegro': 97, 'fiespa': 98, 'fiscro': 99,
'foxspa': 100, 'gadwal': 101, 'gcrfin': 102, 'gnttow': 103, 'gnwtea': 104,
'gockin': 105, 'gocspa': 106, 'goleag': 107, 'grbher3': 108, 'grcfly': 109,
'greegr': 110, 'greroa': 111, 'greyel': 112, 'grhowl': 113, 'grnher': 114,
'grtgra': 115, 'grycat': 116, 'gryfly': 117, 'haiwoo': 118, 'hamfly': 119,
'hergul': 120, 'herthr': 121, 'hoomer': 122, 'hoowar': 123, 'horgre': 124,
'horlar': 125, 'houfin': 126, 'houspa': 127, 'houwre': 128, 'indbun': 129,
'juntit1': 130, 'killde': 131, 'labwoo': 132, 'larspa': 133, 'lazbun': 134,
'leabit': 135, 'leafly': 136, 'leasan': 137, 'lecthr': 138, 'lesgol': 139,
'lesnig': 140, 'lesyel': 141, 'lewwoo': 142, 'linspa': 143, 'lobcur': 144,
'lobdow': 145, 'logshr': 146, 'lotduc': 147, 'louwat': 148, 'macwar': 149,
'magwar': 150, 'mallar3': 151, 'marwre': 152, 'merlin': 153, 'moublu': 154,
'mouchi': 155, 'moudov': 156, 'norcar': 157, 'norfli': 158, 'norhar2': 159,
'normoc': 160, 'norpar': 161, 'norpin': 162, 'norsho': 163, 'norwat': 164,
'nrwswa': 165, 'nutwoo': 166, 'olsfly': 167, 'orcwar': 168, 'osprey': 169,
'ovenbi1': 170, 'palwar': 171, 'pasfly': 172, 'pecsan': 173, 'perfal': 174,
'phaino': 175, 'pibgre': 176, 'pilwoo': 177, 'pingro': 178, 'pinjay': 179,
'pinsis': 180, 'pinwar': 181, 'plsvir': 182, 'prawar': 183, 'purfin': 184,
'pygnut': 185, 'rebmer': 186, 'rebnut': 187, 'rebsap': 188, 'rebwoo': 189,
'redcro': 190, 'redhea': 191, 'reevir1': 192, 'renpha': 193, 'reshaw': 194,
'rethaw': 195, 'rewbla': 196, 'ribgul': 197, 'rinduc': 198, 'robgro': 199,
'rocpig': 200, 'rocwre': 201, 'rthhum': 202, 'ruckin': 203, 'rudduc': 204,
'rufgro': 205, 'rufhum': 206, 'rusbla': 207, 'sagspa1': 208, 'sagthr': 209,
'savspa': 210, 'saypho': 211, 'scatan': 212, 'scoori': 213, 'semplo': 214,
'semsan': 215, 'sheowl': 216, 'shshaw': 217, 'snobun': 218, 'snogoo': 219,
'solsan': 220, 'sonspa': 221, 'sora': 222, 'sposan': 223, 'spotow': 224,
'stejay': 225, 'swahaw': 226, 'swaspa': 227, 'swathr': 228, 'treswa': 229,
'truswa': 230, 'tuftit': 231, 'tunswa': 232, 'veery': 233, 'vesspa': 234,
'vigswa': 235, 'warvir': 236, 'wesblu': 237, 'wesgre': 238, 'weskin': 239,
'wesmea': 240, 'wessan': 241, 'westan': 242, 'wewpew': 243, 'whbnut': 244,
'whcspa': 245, 'whfibi': 246, 'whtspa': 247, 'whtswi': 248, 'wilfly': 249,
'wilsni1': 250, 'wiltur': 251, 'winwre3': 252, 'wlswar': 253, 'wooduc': 254,
'wooscj2': 255, 'woothr': 256, 'y00475': 257, 'yebfly': 258, 'yebsap': 259,
'yehbla': 260, 'yelwar': 261, 'yerwar': 262, 'yetvir': 263
}
CODE2BIRD = {v:k for k,v in BIRD_CODE.items() }
def fe_stats(train, test):
features_g = GENES
features_c = CELLS
for df in train, test:
df['g_sum'] = df[features_g].sum(axis = 1)
 df['g_mean'] = df[features_g].mean(axis = 1)
 df['g_std'] = df[features_g].std(axis = 1)
 df['g_kurt'] = df[features_g].kurtosis(axis = 1)
 df['g_skew'] = df[features_g].skew(axis = 1)
 df['c_sum'] = df[features_c].sum(axis = 1)
 df['c_mean'] = df[features_c].mean(axis = 1)
 df['c_std'] = df[features_c].std(axis = 1)
 df['c_kurt'] = df[features_c].kurtosis(axis = 1)
 df['c_skew'] = df[features_c].skew(axis = 1)
 df['gc_sum'] = df[features_g + features_c].sum(axis = 1)
 df['gc_mean'] = df[features_g + features_c].mean(axis = 1)
 df['gc_std'] = df[features_g + features_c].std(axis = 1)
 df['gc_kurt'] = df[features_g + features_c].kurtosis(axis = 1)
 df['gc_skew'] = df[features_g + features_c].skew(axis = 1)
 df['c52_c42'] = df['c-52'] * df['c-42']
df['c13_c73'] = df['c-13'] * df['c-73']
df['c26_c13'] = df['c-23'] * df['c-13']
df['c33_c6'] = df['c-33'] * df['c-6']
df['c11_c55'] = df['c-11'] * df['c-55']
df['c38_c63'] = df['c-38'] * df['c-63']
df['c38_c94'] = df['c-38'] * df['c-94']
df['c13_c94'] = df['c-13'] * df['c-94']
df['c4_c52'] = df['c-4'] * df['c-52']
df['c4_c42'] = df['c-4'] * df['c-42']
df['c13_c38'] = df['c-13'] * df['c-38']
df['c55_c2'] = df['c-55'] * df['c-2']
df['c55_c4'] = df['c-55'] * df['c-4']
df['c4_c13'] = df['c-4'] * df['c-13']
df['c82_c42'] = df['c-82'] * df['c-42']
df['c66_c42'] = df['c-66'] * df['c-42']
df['c6_c38'] = df['c-6'] * df['c-38']
df['c2_c13'] = df['c-2'] * df['c-13']
df['c62_c42'] = df['c-62'] * df['c-42']
df['c90_c55'] = df['c-90'] * df['c-55']
for feature in features_c:
df[f'{feature}_squared'] = df[feature] ** 2
for feature in gsquarecols:
df[f'{feature}_squared'] = df[feature] ** 2
return train, test
train_features2,test_features2=fe_stats(train_features2,test_features2 )<split>
class ReadAudio(object):
def __init__(
 self 
):
pass
def __call__(self, filename: str)-> Tuple[np.ndarray, int]:
au, sr = librosa.load(filename, sr=None, dtype=np.float64)
 return au, sr
class PreprocessWaveform(object):
def __init__(
 self,
target_sr: int,
normalize: bool = True, 
):
self.target_sr=target_sr
self.normalize=normalize
def __call__(self, wave: np.ndarray, sr: int)-> np.ndarray:
au = librosa.resample(wave, sr, self.target_sr)
 if self.normalize:
au = librosa.util.normalize(au)
 return au
train_features_stats=train_features2.iloc[:,902:]
test_features_stats=test_features2.iloc[:,902:]<concatenate>
class InfernceDataPipelineFileName2Au(object):
def __init__(
 self,
target_sr: bool,
pp_normalize: bool, 
):
self.au_reader = ReadAudio()
self.au_pp = PreprocessWaveform(
 target_sr=target_sr,
normalize=pp_normalize 
)
 def __call__(
 self,
au_path: str 
):
readed_au, readed_sr = self.au_reader(au_path)
 au = self.au_pp(readed_au, readed_sr)
 pp_sr = self.au_pp.target_sr
result = {
'au':au,
'sr':pp_sr,
}
return result
train_features = pd.concat(( train_features, train_features_cluster,train_cluster_pca,train_features_stats), axis=1)
 test_features = pd.concat(( test_features, test_features_cluster,test_cluster_pca,test_features_stats), axis=1 )<merge>
class Test2StepDataset(torch.utils.data.Dataset):
def __init__(
 self,
df: pd.DataFrame,
root_path: str,
data_pipeline_austep: object,
extension: str = '.mp3',
duration: int = 5 
):
self.df = df
self.data_pipeline_austep = data_pipeline_austep
self.root_path = root_path
self.extension = extension
self.duration = duration
self.dumped_features = None
def __len__(self):
return len(self.df)
 def __getitem__(self, idx: int):
sample = self.df.loc[idx, :]
site = sample.site
row_id = sample.row_id
name = sample.audio_id
if self.dumped_features is None or self.dumped_features['name'] != name:
self.dumped_features = {
'name': name,
'features': self.data_pipeline_austep(pjoin(self.root_path, name + self.extension))
}
print(f'{name} feature loaded')
 if site == "site_3":
n_secs = len(self.dumped_features['features']['au'])/ self.dumped_features['features']['sr']
sample_rate = self.dumped_features['features']['sr']
start_sec = 0
specs = []
names = []
sites = []
row_ids = []
while n_secs >= start_sec:
y_batch = self.dumped_features['features']['au']
y_batch = y_batch[start_sec*sample_rate:(start_sec+self.duration)*sample_rate]
y_batch = y_batch.astype(np.float32)
 if len(y_batch)< MIN_SEC * sample_rate:
break
if len(y_batch)< sample_rate * self.duration:
y_pad = np.zeros(sample_rate * self.duration, dtype=np.float32)
 y_pad[:len(y_batch)] = y_batch
y_batch = y_pad
y_batch = torch.from_numpy(y_batch)
 start_sec += self.duration
specs.append(y_batch)
 names.append(name)
 sites.append(sites)
 row_ids.append(row_id)
 return specs, names, sites, row_ids
else:
end_seconds = int(sample.seconds)
 start_seconds = int(end_seconds - self.duration)
 sample_rate = self.dumped_features['features']['sr']
y_batch = self.dumped_features['features']['au']
y_batch = y_batch[start_seconds*sample_rate:end_seconds*sample_rate]
y_batch = y_batch.astype(np.float32)
 if len(y_batch)< sample_rate * self.duration:
y_pad = np.zeros(sample_rate * self.duration, dtype=np.float32)
 y_pad[:len(y_batch)] = y_batch
y_batch = y_pad
y_batch = torch.from_numpy(y_batch)
 return y_batch, name, site, row_id
train = train_features.merge(train_targets_scored, on='sig_id')
 train = train.merge(train_targets_nonscored, on='sig_id')
 train = train.merge(train_drug, on='sig_id')
 train = train[train['cp_type'] != 'ctl_vehicle'].reset_index(drop=True)
 test = test_features[test_features['cp_type'] != 'ctl_vehicle'].reset_index(drop=True )<drop_column>
TEST = Path(".. /input/birdsong-recognition/test_audio" ).exists()
if TEST:
DATA_DIR = Path(".. /input/birdsong-recognition/")
 else:
DATA_DIR = Path(".. /input/birdcall-check/")
 test_df = pd.read_csv(DATA_DIR / "test.csv")
 test_root = DATA_DIR / "test_audio"
test_df.head()
train = train.drop('cp_type', axis=1)
 test = test.drop('cp_type', axis=1 )<define_variables>
test_dataset = Test2StepDataset(
 df=test_df,
root_path=test_root,
data_pipeline_austep=InfernceDataPipelineFileName2Au(
 target_sr=TARGET_SR,
pp_normalize=PP_NORMALIZE 
),
extension='.mp3',
duration=5
)
target_cols = [x for x in train_targets_scored.columns if x != 'sig_id']
aux_target_cols = [x for x in train_targets_nonscored.columns if x != 'sig_id']
all_target_cols = target_cols + aux_target_cols
num_targets = len(target_cols)
 num_aux_targets = len(aux_target_cols)
 num_all_targets = len(all_target_cols)
 print('num_targets: {}'.format(num_targets))
print('num_aux_targets: {}'.format(num_aux_targets))
print('num_all_targets: {}'.format(num_all_targets))<categorify>
def compute_deltas(
 specgram: torch.Tensor,
win_length: int = 5,
mode: str = "replicate" 
)-> torch.Tensor:
r
device = specgram.device
dtype = specgram.dtype
shape = specgram.size()
specgram = specgram.reshape(1, -1, shape[-1])
 assert win_length >= 3
n =(win_length - 1)// 2
denom = n *(n + 1)*(2 * n + 1)/ 3
specgram = torch.nn.functional.pad(specgram,(n, n), mode=mode)
 kernel = torch.arange(-n, n + 1, 1, device=device, dtype=dtype ).repeat(specgram.shape[1], 1, 1)
 output = torch.nn.functional.conv1d(specgram, kernel, groups=specgram.shape[1])/ denom
output = output.reshape(shape)
 return output
def make_delta(
 input_tensor: torch.Tensor 
):
input_tensor = input_tensor.transpose(3,2)
 input_tensor = compute_deltas(input_tensor)
 input_tensor = input_tensor.transpose(3,2)
 return input_tensor
class MoADataset:
def __init__(self, features, targets):
self.features = features
self.targets = targets
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float),
'y' : torch.tensor(self.targets[idx, :], dtype=torch.float)
 }
return dct
class TestDataset:
def __init__(self, features):
self.features = features
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float)
 }
return dct<train_model>
def _resnext(block, layers, pretrained, progress, **kwargs):
return ResNet(block, layers, **kwargs)
 def resnext50_32x4d_swsl(progress=True, **kwargs):

 kwargs['groups'] = 32
kwargs['width_per_group'] = 4
return _resnext(Bottleneck, [3, 4, 6, 3], True, progress, **kwargs)
 def resnext101_32x4d_swsl(progress=True, **kwargs):

 kwargs['groups'] = 32
kwargs['width_per_group'] = 4
return _resnext(Bottleneck, [3, 4, 23, 3], True, progress, **kwargs )
def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):
model.train()
final_loss = 0
for data in dataloader:
optimizer.zero_grad()
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
scheduler.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 return final_loss
def valid_fn(model, loss_fn, dataloader, device):
model.eval()
final_loss = 0
valid_preds = []
for data in dataloader:
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 return final_loss, valid_preds
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
inputs = data['x'].to(device)
 with torch.no_grad() :
outputs = model(inputs)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds<normalization>
EFFNETB6_EMB_DIM = 2304
EFFNETB5_EMB_DIM = 2048
EFFNETB4_EMB_DIM = 1792
EFFNETB3_EMB_DIM = 1536
EFFNETB1_EMB_DIM = 1280
RESNEST50_FAST_EMB_DIM = 2048
RESNEXT50_EMB_DIM = 2048
RESNEXT101_EMB_DIM = 2048
EPS = 1e-6
class SpectralEffnet(nn.Module):
def __init__(
 self,
sample_rate: int,
window_size: int,
hop_size: int,
mel_bins: int,
fmin: int,
fmax: int,
top_db: float,
classes_num: int,
encoder_type: str,
hidden_dims: int,
first_dropout_rate: float,
second_dropout_rate: float,
use_pretrained_encoder: bool,
classifier_type: str,
use_spectral_cutout: bool,
spec_aggreagation: str,
use_loudness: bool,
use_spectral_centroid: bool,
device: str,
use_sigmoid: bool = True, 
):
super().__init__()
window = 'hann'
center = True
pad_mode = 'reflect'
ref = 1.0
amin = 1e-10
self.interpolate_ratio = 32
self.spectrogram_extractor = Spectrogram(
 n_fft=window_size,
hop_length=hop_size,
win_length=window_size,
window=window,
center=center,
pad_mode=pad_mode,
freeze_parameters=True)
 self.logmel_extractor = LogmelFilterBank(
 sr=sample_rate,
n_fft=window_size,
n_mels=mel_bins,
fmin=fmin,
fmax=fmax,
ref=ref,
amin=amin,
top_db=top_db,
freeze_parameters=True)
 self.spec_augmenter = SpecAugmentation(
 time_drop_width=64,
time_stripes_num=2,
freq_drop_width=8,
freq_stripes_num=2)
 if use_loudness:
self.loudness_bn = nn.BatchNorm1d(1)
 self.loudness_extractor = Loudness(
 sr=sample_rate,
n_fft=window_size,
min_db=120,
device=device 
)
 if use_spectral_centroid:
self.spectral_centroid_bn = nn.BatchNorm1d(1)
 self.bn0 = nn.BatchNorm2d(mel_bins)
 if spec_aggreagation in ['conv1', 'repeat3', 'deltas', 'conv3', 'time_freq_encoding']:
self.spec_aggreagation = spec_aggreagation
else:
raise ValueError('Invalid spec_aggreagation')
 if encoder_type == 'effnet1':
self.encoder = geffnet.tf_efficientnet_b1_ns(pretrained=use_pretrained_encoder)
 self.encoder.classifier = nn.Identity()
if self.spec_aggreagation == 'conv1':
self.encoder.conv_stem = Conv2dSame(
 in_channels=1,
out_channels=self.encoder.conv_stem.out_channels,
kernel_size=self.encoder.conv_stem.kernel_size,
stride=self.encoder.conv_stem.stride,
bias=self.encoder.conv_stem.bias 
)
 nn_embed_size = EFFNETB1_EMB_DIM
elif encoder_type == 'effnet3':
self.encoder = geffnet.tf_efficientnet_b3_ns(pretrained=use_pretrained_encoder)
 self.encoder.classifier = nn.Identity()
if self.spec_aggreagation == 'conv1':
self.encoder.conv_stem = Conv2dSame(
 in_channels=1,
out_channels=self.encoder.conv_stem.out_channels,
kernel_size=self.encoder.conv_stem.kernel_size,
stride=self.encoder.conv_stem.stride,
bias=self.encoder.conv_stem.bias 
)
 nn_embed_size = EFFNETB3_EMB_DIM
elif encoder_type == 'effnet4':
self.encoder = geffnet.tf_efficientnet_b4_ns(pretrained=use_pretrained_encoder)
 self.encoder.classifier = nn.Identity()
if self.spec_aggreagation == 'conv1':
self.encoder.conv_stem = Conv2dSame(
 in_channels=1,
out_channels=self.encoder.conv_stem.out_channels,
kernel_size=self.encoder.conv_stem.kernel_size,
stride=self.encoder.conv_stem.stride,
bias=self.encoder.conv_stem.bias 
)
 nn_embed_size = EFFNETB4_EMB_DIM
elif encoder_type == 'effnet5':
self.encoder = geffnet.tf_efficientnet_b5_ns(pretrained=use_pretrained_encoder)
 self.encoder.classifier = nn.Identity()
if self.spec_aggreagation == 'conv1':
self.encoder.conv_stem = Conv2dSame(
 in_channels=1,
out_channels=self.encoder.conv_stem.out_channels,
kernel_size=self.encoder.conv_stem.kernel_size,
stride=self.encoder.conv_stem.stride,
bias=self.encoder.conv_stem.bias 
)
 nn_embed_size = EFFNETB5_EMB_DIM
elif encoder_type == 'effnet6':
self.encoder = geffnet.tf_efficientnet_b6_ns(pretrained=use_pretrained_encoder)
 self.encoder.classifier = nn.Identity()
if self.spec_aggreagation == 'conv1':
self.encoder.conv_stem = Conv2dSame(
 in_channels=1,
out_channels=self.encoder.conv_stem.out_channels,
kernel_size=self.encoder.conv_stem.kernel_size,
stride=self.encoder.conv_stem.stride,
bias=self.encoder.conv_stem.bias 
)
 nn_embed_size = EFFNETB6_EMB_DIM
elif encoder_type == 'resnest50_fast_1s1x64d':
if self.spec_aggreagation == 'conv1':
raise ValueError('Invalid spec_aggreagation for this encoder')
 self.encoder = resnest50_fast_1s1x64d(pretrained=use_pretrained_encoder)
 self.encoder.fc = nn.Identity()
nn_embed_size = RESNEST50_FAST_EMB_DIM
elif encoder_type == 'resnext50_32x4d_swsl':
self.encoder = resnext50_32x4d_swsl()
if use_pretrained_encoder:
self.encoder.load_state_dict(
 torch.hub.load('facebookresearch/semi-supervised-ImageNet1K-models', 'resnext50_32x4d_swsl' ).state_dict() 
)
 self.encoder.fc = nn.Identity()
if self.spec_aggreagation == 'conv1':
self.encoder.conv1 = nn.Conv2d(
 in_channels=1,
out_channels=self.encoder.conv1.out_channels,
kernel_size=self.encoder.conv1.kernel_size,
stride=self.encoder.conv1.stride,
bias=self.encoder.conv1.bias 
)
 nn_embed_size = RESNEXT50_EMB_DIM
elif encoder_type == 'resnext101_32x4d_swsl':
self.encoder = resnext101_32x4d_swsl()
if use_pretrained_encoder:
self.encoder.load_state_dict(
 torch.hub.load('facebookresearch/semi-supervised-ImageNet1K-models', 'resnext101_32x4d_swsl' ).state_dict() 
)
 self.encoder.fc = nn.Identity()
if self.spec_aggreagation == 'conv1':
self.encoder.conv1 = nn.Conv2d(
 in_channels=1,
out_channels=self.encoder.conv1.out_channels,
kernel_size=self.encoder.conv1.kernel_size,
stride=self.encoder.conv1.stride,
bias=self.encoder.conv1.bias 
)
 nn_embed_size = RESNEXT101_EMB_DIM
else:
raise ValueError(f'{encoder_type} is invalid model_type')
 if classifier_type == 'relu':
self.classifier = nn.Sequential(
 nn.Linear(nn_embed_size, hidden_dims), nn.ReLU() , nn.Dropout(p=first_dropout_rate),
nn.Linear(hidden_dims, hidden_dims), nn.ReLU() , nn.Dropout(p=second_dropout_rate),
nn.Linear(hidden_dims, classes_num)
)
 elif classifier_type == 'elu':
self.classifier = nn.Sequential(
 nn.Dropout(first_dropout_rate),
nn.Linear(nn_embed_size, hidden_dims),
nn.ELU() ,
nn.Dropout(second_dropout_rate),
nn.Linear(hidden_dims, classes_num)
)
 elif classifier_type == 'dima':
self.classifier = nn.Sequential(
 nn.BatchNorm1d(nn_embed_size),
nn.Linear(nn_embed_size, hidden_dims),
nn.BatchNorm1d(hidden_dims),
nn.PReLU(hidden_dims),
nn.Dropout(p=second_dropout_rate),
nn.Linear(hidden_dims, classes_num)
)
 elif classifier_type == 'prelu':
self.classifier = nn.Sequential(
 nn.Dropout(first_dropout_rate),
nn.Linear(nn_embed_size, hidden_dims),
nn.PReLU(hidden_dims),
nn.Dropout(p=second_dropout_rate),
nn.Linear(hidden_dims, classes_num)
)
 elif classifier_type == 'multiscale_relu':
self.big_dropout = nn.Dropout(p=0.5)
 self.classifier = nn.Sequential(
 nn.Linear(nn_embed_size, hidden_dims), nn.ReLU() , nn.Dropout(p=first_dropout_rate),
nn.Linear(hidden_dims, hidden_dims), nn.ReLU() , nn.Dropout(p=second_dropout_rate),
nn.Linear(hidden_dims, classes_num)
)
 else:
raise ValueError("Invalid classifier_type")
 self.classifier_type = classifier_type
self.use_spectral_cutout = use_spectral_cutout
self.use_sigmoid = use_sigmoid
self.use_spectral_centroid = use_spectral_centroid
self.use_loudness = use_loudness
self.encoder_type = encoder_type
self.device = device
self.mel_bins = mel_bins
self.to(self.device)
 def _add_frequency_encoding(self, x):
n, d, h, w = x.size()
vertical = torch.linspace(-1, 1, w, device=x.device ).view(1, 1, 1, -1)
 vertical = vertical.repeat(n, 1, h, 1)
 return vertical
def _add_time_encoding(self, x):
n, d, h, w = x.size()
horizontal = torch.linspace(-1, 1, h, device=x.device ).view(1, 1, -1, 1)
 horizontal = horizontal.repeat(n, 1, 1, w)
 return horizontal
def preprocess(self, input):
x = self.spectrogram_extractor(input)
 additional_features = []
if self.use_loudness:
loudness = self.loudness_extractor(x)
 loudness = self.loudness_bn(loudness)
 loudness = loudness.unsqueeze(-1)
 loudness = loudness.repeat(1,1,1,self.mel_bins)
 additional_features.append(loudness)
 if self.use_spectral_centroid:
spectral_centroid = x.mean(-1)
 spectral_centroid = self.spectral_centroid_bn(spectral_centroid)
 spectral_centroid = spectral_centroid.unsqueeze(-1)
 spectral_centroid = spectral_centroid.repeat(1,1,1,self.mel_bins)
 additional_features.append(spectral_centroid)
 x = self.logmel_extractor(x)
 if self.training and self.use_spectral_cutout:
x = self.spec_augmenter(x)
 frames_num = x.shape[2]
x = x.transpose(1, 3)
 x = self.bn0(x)
 x = x.transpose(1, 3)
 if len(additional_features)> 0:
additional_features.append(x)
 x = torch.cat(additional_features, dim=1)
 if self.spec_aggreagation == 'repeat3':
x = torch.cat([x,x,x], dim=1)
 elif self.spec_aggreagation == 'deltas':
delta_1 = make_delta(x)
 delta_2 = make_delta(delta_1)
 x = torch.cat([x,delta_1,delta_2], dim=1)
 elif self.spec_aggreagation == 'time_freq_encoding':
freq_encode = self._add_frequency_encoding(x)
 time_encode = self._add_time_encoding(x)
 x = torch.cat([x, freq_encode, time_encode], dim=1)
 elif self.spec_aggreagation in ['conv1','conv3']:
pass
return x, frames_num
def forward(self, input):

 x, _ = self.preprocess(input)
 x = self.encoder(x)
 if self.classifier_type == 'multiscale_relu':
logits = torch.mean(torch.stack([self.classifier(self.big_dropout(x)) for _ in range(5)],dim=0),dim=0)
 else:
logits = self.classifier(x)
 if self.use_sigmoid:
return torch.sigmoid(logits)
 else:
return logits
class SmoothBCEwLogits(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=0.0):
super().__init__(weight=weight, reduction=reduction)
 self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
@staticmethod
def _smooth(targets:torch.Tensor, n_labels:int, smoothing=0.0):
assert 0 <= smoothing < 1
with torch.no_grad() :
targets = targets *(1.0 - smoothing)+ 0.5 * smoothing
return targets
def forward(self, inputs, targets):
targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1),
self.smoothing)
 loss = F.binary_cross_entropy_with_logits(inputs, targets,self.weight)
 if self.reduction == 'sum':
loss = loss.sum()
elif self.reduction == 'mean':
loss = loss.mean()
return loss<choose_model_class>
def load_chkp(
 path: str,
nn_model: nn.Module 
):
chckp = torch.load(path, map_location='cpu')
 nn_model.load_state_dict(chckp)
 return nn_model
def initalize_spectral_model(
 model_config: Mapping[str, Any],
chkp_path: str,
m_device: str 
):
temp_model = SpectralEffnet(
 **model_config,
device=m_device 
)
 temp_model = load_chkp(chkp_path, temp_model)
 temp_model.eval()
return temp_model
class Model(nn.Module):
def __init__(self, num_features, num_targets):
super(Model, self ).__init__()
self.hidden_size = [1500, 1250, 1000, 750]
self.dropout_value = [0.5, 0.35, 0.3, 0.25]
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dense1 = nn.Linear(num_features, self.hidden_size[0])
 self.batch_norm2 = nn.BatchNorm1d(self.hidden_size[0])
 self.dropout2 = nn.Dropout(self.dropout_value[0])
 self.dense2 = nn.Linear(self.hidden_size[0], self.hidden_size[1])
 self.batch_norm3 = nn.BatchNorm1d(self.hidden_size[1])
 self.dropout3 = nn.Dropout(self.dropout_value[1])
 self.dense3 = nn.Linear(self.hidden_size[1], self.hidden_size[2])
 self.batch_norm4 = nn.BatchNorm1d(self.hidden_size[2])
 self.dropout4 = nn.Dropout(self.dropout_value[2])
 self.dense4 = nn.Linear(self.hidden_size[2], self.hidden_size[3])
 self.batch_norm5 = nn.BatchNorm1d(self.hidden_size[3])
 self.dropout5 = nn.Dropout(self.dropout_value[3])
 self.dense5 = nn.utils.weight_norm(nn.Linear(self.hidden_size[3], num_targets))
def forward(self, x):
x = self.batch_norm1(x)
 x = F.leaky_relu(self.dense1(x))
x = self.batch_norm2(x)
 x = self.dropout2(x)
 x = F.leaky_relu(self.dense2(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = F.leaky_relu(self.dense3(x))
x = self.batch_norm4(x)
 x = self.dropout4(x)
 x = F.leaky_relu(self.dense4(x))
x = self.batch_norm5(x)
 x = self.dropout5(x)
 x = self.dense5(x)
 return x
class LabelSmoothingLoss(nn.Module):
def __init__(self, classes, smoothing=0.0, dim=-1):
super(LabelSmoothingLoss, self ).__init__()
self.confidence = 1.0 - smoothing
self.smoothing = smoothing
self.cls = classes
self.dim = dim
def forward(self, pred, target):
pred = pred.log_softmax(dim=self.dim)
 with torch.no_grad() :
true_dist = torch.zeros_like(pred)
 true_dist.fill_(self.smoothing /(self.cls - 1))
true_dist.scatter_(1, target.data.unsqueeze(1), self.confidence)
 return torch.mean(torch.sum(-true_dist * pred, dim=self.dim))<choose_model_class>
NEW_EFFNET3 = {
"sample_rate": TARGET_SR,
"window_size": 2048,
"hop_size": 256,
"mel_bins": 128,
"fmin": 20,
"fmax": 16000,
"top_db":80.0,
"classes_num": len(BIRD_CODE),
"encoder_type": 'effnet3',
"hidden_dims": 1024,
"first_dropout_rate": 0.2,
"second_dropout_rate": 0.2,
"use_spectral_cutout": False,
"spec_aggreagation": "deltas",
"use_pretrained_encoder": False,
"use_sigmoid": True,
"classifier_type": "relu",
"use_loudness": False,
"use_spectral_centroid": False,
}
NEW_EFFNET3_MD = {
"sample_rate": TARGET_SR,
"window_size": 2048,
"hop_size": 256,
"mel_bins": 128,
"fmin": 20,
"fmax": 16000,
"top_db":80.0,
"classes_num": len(BIRD_CODE),
"encoder_type": 'effnet3',
"hidden_dims": 1024,
"first_dropout_rate": 0.2,
"second_dropout_rate": 0.2,
"use_spectral_cutout": False,
"spec_aggreagation": "deltas",
"use_pretrained_encoder": False,
"use_sigmoid": True,
"classifier_type": "multiscale_relu",
"use_loudness": False,
"use_spectral_centroid": False,
}
NEW_EFFNET4 = {
"sample_rate": TARGET_SR,
"window_size": 2048,
"hop_size": 256,
"mel_bins": 128,
"fmin": 20,
"fmax": 16000,
"top_db":80.0,
"classes_num": len(BIRD_CODE),
"encoder_type": 'effnet4',
"hidden_dims": 1024,
"first_dropout_rate": 0.2,
"second_dropout_rate": 0.2,
"use_spectral_cutout": False,
"spec_aggreagation": "deltas",
"use_pretrained_encoder": False,
"use_sigmoid": True,
"classifier_type": "relu",
"use_loudness": False,
"use_spectral_centroid": False,
}
NEW_EFFNET4_MD = {
"sample_rate": TARGET_SR,
"window_size": 2048,
"hop_size": 256,
"mel_bins": 128,
"fmin": 20,
"fmax": 16000,
"top_db":80.0,
"classes_num": len(BIRD_CODE),
"encoder_type": 'effnet4',
"hidden_dims": 1024,
"first_dropout_rate": 0.2,
"second_dropout_rate": 0.2,
"use_spectral_cutout": False,
"spec_aggreagation": "deltas",
"use_pretrained_encoder": False,
"use_sigmoid": True,
"classifier_type": "multiscale_relu",
"use_loudness": False,
"use_spectral_centroid": False,
}
NEW_EFFNET5_MD = {
"sample_rate": TARGET_SR,
"window_size": 2048,
"hop_size": 256,
"mel_bins": 128,
"fmin": 20,
"fmax": 16000,
"top_db":80.0,
"classes_num": len(BIRD_CODE),
"encoder_type": 'effnet5',
"hidden_dims": 1024,
"first_dropout_rate": 0.2,
"second_dropout_rate": 0.2,
"use_spectral_cutout": False,
"spec_aggreagation": "deltas",
"use_pretrained_encoder": False,
"use_sigmoid": True,
"classifier_type": "multiscale_relu",
"use_loudness": False,
"use_spectral_centroid": False,
}
NEW_EFFNET3_MD_TFE = {
"sample_rate": TARGET_SR,
"window_size": 2048,
"hop_size": 256,
"mel_bins": 128,
"fmin": 20,
"fmax": 16000,
"top_db":80.0,
"classes_num": len(BIRD_CODE),
"encoder_type": 'effnet3',
"hidden_dims": 1024,
"first_dropout_rate": 0.2,
"second_dropout_rate": 0.2,
"use_spectral_cutout": False,
"spec_aggreagation": "time_freq_encoding",
"use_pretrained_encoder": False,
"use_sigmoid": True,
"classifier_type": "multiscale_relu",
"use_loudness": False,
"use_spectral_centroid": False,
}
models = [
initalize_spectral_model(NEW_EFFNET3_MD, '.. /input/cornell-birds-models/effnet3_multiscalereluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave075_xeno_canto_swa_f1_test_median_fold_0.pt', DEVICE),
initalize_spectral_model(NEW_EFFNET3_MD, '.. /input/cornell-birds-models/effnet3_multiscalereluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave075_xeno_canto_swa_f1_test_median_fold_1.pt', DEVICE),
initalize_spectral_model(NEW_EFFNET3_MD, '.. /input/cornell-birds-models/effnet3_multiscalereluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave075_xeno_canto_swa_f1_test_median_fold_2.pt', DEVICE),
initalize_spectral_model(NEW_EFFNET3_MD, '.. /input/cornell-birds-models/effnet3_multiscalereluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave075_xeno_canto_swa_f1_test_median_fold_3.pt', DEVICE),
initalize_spectral_model(NEW_EFFNET3_MD, '.. /input/cornell-birds-models/effnet3_multiscalereluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave075_xeno_canto_swa_f1_test_median_fold_4.pt', DEVICE),
initalize_spectral_model(NEW_EFFNET4, '.. /input/cornell-birds-models/effnet4_reluclas_plato_deltas_50bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave075_xeno_canto_swa_f1_test_median_fold_0.pt', DEVICE),
initalize_spectral_model(NEW_EFFNET4, '.. /input/cornell-birds-models/effnet4_reluclas_plato_deltas_50bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave075_xeno_canto_swa_f1_test_median_fold_1.pt', DEVICE),
initalize_spectral_model(NEW_EFFNET4, '.. /input/cornell-birds-models/effnet4_reluclas_plato_deltas_50bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave075_xeno_canto_swa_f1_test_median_fold_2.pt', DEVICE),
initalize_spectral_model(NEW_EFFNET4, '.. /input/cornell-birds-models/effnet4_reluclas_plato_deltas_50bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave075_xeno_canto_swa_f1_test_median_fold_3.pt', DEVICE),
initalize_spectral_model(NEW_EFFNET4, '.. /input/cornell-birds-models/effnet4_reluclas_plato_deltas_50bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave075_xeno_canto_swa_f1_test_median_fold_4.pt', DEVICE),
initalize_spectral_model(NEW_EFFNET3, '.. /input/cornell-birds-models/effnet3_reluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave075_xeno_canto_swa_f1_test_median_fold_0.pt', DEVICE),
initalize_spectral_model(NEW_EFFNET3, '.. /input/cornell-birds-models/effnet3_reluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave075_xeno_canto_swa_f1_test_median_fold_1.pt', DEVICE),
initalize_spectral_model(NEW_EFFNET3, '.. /input/cornell-birds-models/effnet3_reluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave075_xeno_canto_swa_f1_test_median_fold_2.pt', DEVICE),
initalize_spectral_model(NEW_EFFNET3, '.. /input/cornell-birds-models/effnet3_reluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave075_xeno_canto_swa_f1_test_median_fold_3.pt', DEVICE),
initalize_spectral_model(NEW_EFFNET3, '.. /input/cornell-birds-models/effnet3_reluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave075_xeno_canto_swa_f1_test_median_fold_4.pt', DEVICE),
initalize_spectral_model(NEW_EFFNET3, '.. /input/cornell-birds-models/effnet3_reluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave05_ls005_swa_f1_test_median_fold_0.pt', DEVICE),
initalize_spectral_model(NEW_EFFNET3, '.. /input/cornell-birds-models/effnet3_reluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave05_ls005_swa_f1_test_median_fold_1.pt', DEVICE),
initalize_spectral_model(NEW_EFFNET3, '.. /input/cornell-birds-models/effnet3_reluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave05_ls005_swa_f1_test_median_fold_2.pt', DEVICE),
initalize_spectral_model(NEW_EFFNET3, '.. /input/cornell-birds-models/effnet3_reluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave05_ls005_swa_f1_test_median_fold_3.pt', DEVICE),
initalize_spectral_model(NEW_EFFNET3, '.. /input/cornell-birds-models/effnet3_reluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave05_ls005_swa_f1_test_median_fold_4.pt', DEVICE),
initalize_spectral_model(NEW_EFFNET3, '.. /input/cornell-birds-models/effnet3_reluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave075_swa_f1_test_median_fold_0.pt', DEVICE),
initalize_spectral_model(NEW_EFFNET3, '.. /input/cornell-birds-models/effnet3_reluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave075_swa_f1_test_median_fold_1.pt', DEVICE),
initalize_spectral_model(NEW_EFFNET3, '.. /input/cornell-birds-models/effnet3_reluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave075_swa_f1_test_median_fold_2.pt', DEVICE),
initalize_spectral_model(NEW_EFFNET3, '.. /input/cornell-birds-models/effnet3_reluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave075_swa_f1_test_median_fold_3.pt', DEVICE),
initalize_spectral_model(NEW_EFFNET3, '.. /input/cornell-birds-models/effnet3_reluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave075_swa_f1_test_median_fold_4.pt', DEVICE),
initalize_spectral_model(NEW_EFFNET5_MD, '.. /input/cornell-birds-models/effnet5_multiscalereluclas_plato_deltas_32bs_2acum_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave075_xeno_canto_swa_f1_test_median_fold_0.pt', DEVICE),
initalize_spectral_model(NEW_EFFNET5_MD, '.. /input/cornell-birds-models/effnet5_multiscalereluclas_plato_deltas_32bs_2acum_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave075_xeno_canto_swa_f1_test_median_fold_1.pt', DEVICE),
initalize_spectral_model(NEW_EFFNET5_MD, '.. /input/cornell-birds-models/effnet5_multiscalereluclas_plato_deltas_32bs_2acum_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave075_xeno_canto_swa_f1_test_median_fold_2.pt', DEVICE),
initalize_spectral_model(NEW_EFFNET5_MD, '.. /input/cornell-birds-models/effnet5_multiscalereluclas_plato_deltas_32bs_2acum_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave075_xeno_canto_swa_f1_test_median_fold_3.pt', DEVICE),
initalize_spectral_model(NEW_EFFNET5_MD, '.. /input/cornell-birds-models/effnet5_multiscalereluclas_plato_deltas_32bs_2acum_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_mixupsumwave075_xeno_canto_swa_f1_test_median_fold_4.pt', DEVICE),
initalize_spectral_model(NEW_EFFNET3, '.. /input/cornell-birds-models/effnet3_reluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_ls005_swa_f1_test_median_fold_0.pt', DEVICE),
initalize_spectral_model(NEW_EFFNET3, '.. /input/cornell-birds-models/effnet3_reluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_ls005_swa_f1_test_median_fold_1.pt', DEVICE),
initalize_spectral_model(NEW_EFFNET3, '.. /input/cornell-birds-models/effnet3_reluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_ls005_swa_f1_test_median_fold_2.pt', DEVICE),
initalize_spectral_model(NEW_EFFNET3, '.. /input/cornell-birds-models/effnet3_reluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_ls005_swa_f1_test_median_fold_3.pt', DEVICE),
initalize_spectral_model(NEW_EFFNET3, '.. /input/cornell-birds-models/effnet3_reluclas_plato_deltas_64bs_001lr_biggerfft_trackmap_energytrimming_topdb80_firstaugs_secondarylabels_ls005_swa_f1_test_median_fold_4.pt', DEVICE),
]
class FineTuneScheduler:
def __init__(self, epochs):
self.epochs = epochs
self.epochs_per_step = 0
self.frozen_layers = []
def copy_without_top(self, model, num_features, num_targets, num_targets_new):
self.frozen_layers = []
model_new = Model(num_features, num_targets)
 model_new.load_state_dict(model.state_dict())
 for name, param in model_new.named_parameters() :
layer_index = name.split('.')[0][-1]
if layer_index == 5:
continue
param.requires_grad = False
if layer_index not in self.frozen_layers:
self.frozen_layers.append(layer_index)
 self.epochs_per_step = self.epochs // len(self.frozen_layers)
 model_new.batch_norm5 = nn.BatchNorm1d(model_new.hidden_size[3])
 model_new.dropout5 = nn.Dropout(model_new.dropout_value[3])
 model_new.dense5 = nn.utils.weight_norm(nn.Linear(model_new.hidden_size[-1], num_targets_new))
model_new.to(DEVICE)
 return model_new
def step(self, epoch, model):
if len(self.frozen_layers)== 0:
return
if epoch % self.epochs_per_step == 0:
last_frozen_index = self.frozen_layers[-1]
for name, param in model.named_parameters() :
layer_index = name.split('.')[0][-1]
if layer_index == last_frozen_index:
param.requires_grad = True
del self.frozen_layers[-1]<categorify>
print(f"Models in Blend: {len(models)}" )
def process_data(data):
data = pd.get_dummies(data, columns=['cp_time','cp_dose'])
 return data<define_variables>
batch = []
names = []
probs = []
for i in tqdm(range(len(test_dataset))):
sample_spec, _, _, sample_id = test_dataset[i]
if isinstance(sample_spec, list):
batch += sample_spec
else:
batch.append(sample_spec)
 if isinstance(sample_id, list):
names += sample_id
else:
names.append(sample_id)
 if len(batch)>= BATCH_SIZE or i ==(len(test_dataset)- 1):
with torch.no_grad() :
if len(batch)> BATCH_SIZE + 1:
n_steps = math.ceil(len(batch)/ BATCH_SIZE)
 for step in range(n_steps):
small_batch = batch[step*BATCH_SIZE:(step+1)*BATCH_SIZE]
small_batch = torch.stack(small_batch ).to(DEVICE ).float()
batch_probs = sum(m(small_batch ).detach().cpu() for m in models)/ len(models)
 batch_probs = batch_probs.numpy()
probs.append(batch_probs)
 else:
batch = torch.stack(batch ).to(DEVICE ).float()
batch_probs = sum(m(batch ).detach().cpu() for m in models)/ len(models)
 batch_probs = batch_probs.numpy()
probs.append(batch_probs)
 batch = []
feature_cols = [c for c in process_data(train ).columns if c not in all_target_cols]
feature_cols = [c for c in feature_cols if c not in ['kfold', 'sig_id', 'drug_id']]
num_features = len(feature_cols)
 num_features<define_variables>
probs = np.concatenate(probs)
 result_df = pd.DataFrame({
'row_id':names,
'birds':[probs[i] for i in range(probs.shape[0])]
})
 result_df = result_df.groupby('row_id')['birds'].apply(lambda x: np.stack(x ).max(0)).reset_index()
result_df
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS = 24
BATCH_SIZE = 128
WEIGHT_DECAY = {'ALL_TARGETS': 1e-5, 'SCORED_ONLY': 3e-6}
MAX_LR = {'ALL_TARGETS': 1e-2, 'SCORED_ONLY': 3e-3}
DIV_FACTOR = {'ALL_TARGETS': 1e3, 'SCORED_ONLY': 1e2}
PCT_START = 0.1<choose_model_class>
def probs2names(
 probs_array: np.ndarray,
threshold: int = 0.5,
max_birds_to_take: Optional[int] = None,
idx2bird_mapping: Mapping[int, str] = CODE2BIRD 
):
accepted_indices = np.where(probs_array > threshold)[0]
if max_birds_to_take is not None:
accepted_probs = probs_array[accepted_indices]
accepted_probs_indices = np.argsort(-accepted_probs)
 accepted_indices = accepted_indices[accepted_probs_indices]
accepted_indices = accepted_indices[:max_birds_to_take]
if len(accepted_indices)== 0:
return 'nocall'
else:
return ' '.join([idx2bird_mapping[idx] for idx in accepted_indices] )
model = Model(num_features, num_all_targets)
 model<define_search_model>
result_df['birds'] = result_df['birds'].apply(lambda x: probs2names(
 x,
threshold=SIGMOID_THRESH,
max_birds_to_take=MAX_BIRDS 
))
def make_cv_folds(train, SEEDS, NFOLDS, DRUG_THRESH):
vc = train.drug_id.value_counts()
vc1 = vc.loc[vc <= DRUG_THRESH].index.sort_values()
vc2 = vc.loc[vc > DRUG_THRESH].index.sort_values()
for seed_id in range(SEEDS):
kfold_col = 'kfold_{}'.format(seed_id)
 dct1 = {}
dct2 = {}
skf = MultilabelStratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=seed_id)
 tmp = train.groupby('drug_id')[target_cols].mean().loc[vc1]
for fold,(idxT, idxV)in enumerate(skf.split(tmp, tmp[target_cols])) :
dd = {k: fold for k in tmp.index[idxV].values}
dct1.update(dd)
 skf = MultilabelStratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=seed_id)
 tmp = train.loc[train.drug_id.isin(vc2)].reset_index(drop=True)
 for fold,(idxT, idxV)in enumerate(skf.split(tmp, tmp[target_cols])) :
dd = {k: fold for k in tmp.sig_id[idxV].values}
dct2.update(dd)
 train[kfold_col] = train.drug_id.map(dct1)
 train.loc[train[kfold_col].isna() , kfold_col] = train.loc[train[kfold_col].isna() , 'sig_id'].map(dct2)
 train[kfold_col] = train[kfold_col].astype('int8')
 return train
SEEDS = 7
NFOLDS = 7
DRUG_THRESH = 18
train = make_cv_folds(train, SEEDS, NFOLDS, DRUG_THRESH)
 train.head()<train_model>
submission = test_df.merge(result_df, on='row_id', how='left')[['row_id','birds']]
submission
def run_training(fold_id, seed_id):
seed_everything(seed_id)
 train_ = process_data(train)
 test_ = process_data(test)
 kfold_col = f'kfold_{seed_id}'
trn_idx = train_[train_[kfold_col] != fold_id].index
val_idx = train_[train_[kfold_col] == fold_id].index
train_df = train_[train_[kfold_col] != fold_id].reset_index(drop=True)
 valid_df = train_[train_[kfold_col] == fold_id].reset_index(drop=True)
 def train_model(model, tag_name, target_cols_now, fine_tune_scheduler=None):
x_train, y_train = train_df[feature_cols].values, train_df[target_cols_now].values
x_valid, y_valid = valid_df[feature_cols].values, valid_df[target_cols_now].values
train_dataset = MoADataset(x_train, y_train)
 valid_dataset = MoADataset(x_valid, y_valid)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 optimizer = torch.optim.Adam(model.parameters() , lr=1e-3, weight_decay=WEIGHT_DECAY[tag_name])
 scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer,
steps_per_epoch=len(trainloader),
pct_start=PCT_START,
div_factor=DIV_FACTOR[tag_name],
max_lr=MAX_LR[tag_name],
epochs=EPOCHS)
 loss_fn = nn.BCEWithLogitsLoss()
loss_tr = SmoothBCEwLogits(smoothing=0.0015)
 oof = np.zeros(( len(train), len(target_cols_now)))
 best_loss = np.inf
for epoch in range(EPOCHS):
if fine_tune_scheduler is not None:
fine_tune_scheduler.step(epoch, model)
 train_loss = train_fn(model, optimizer, scheduler, loss_tr, trainloader, DEVICE)
 valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 print(f"SEED: {seed_id}, FOLD: {fold_id}, {tag_name}, EPOCH: {epoch}, train_loss: {train_loss:.6f}, valid_loss: {valid_loss:.6f}")
 if np.isnan(valid_loss):
break
if valid_loss < best_loss:
best_loss = valid_loss
oof[val_idx] = valid_preds
torch.save(model.state_dict() , f"{tag_name}_FOLD{fold_id}_.pth")
 return oof
fine_tune_scheduler = FineTuneScheduler(EPOCHS)
 pretrained_model = Model(num_features, num_all_targets)
 pretrained_model.to(DEVICE)
 train_model(pretrained_model, 'ALL_TARGETS', all_target_cols)
 pretrained_model = Model(num_features, num_all_targets)
 pretrained_model.load_state_dict(torch.load(f"ALL_TARGETS_FOLD{fold_id}_.pth"))
pretrained_model.to(DEVICE)
 final_model = fine_tune_scheduler.copy_without_top(pretrained_model, num_features, num_all_targets, num_targets)
 oof = train_model(final_model, 'SCORED_ONLY', target_cols, fine_tune_scheduler)
 model = Model(num_features, num_targets)
 model.load_state_dict(torch.load(f"SCORED_ONLY_FOLD{fold_id}_.pth"))
model.to(DEVICE)
 x_test = test_[feature_cols].values
testdataset = TestDataset(x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 predictions = np.zeros(( len(test_), num_targets))
predictions = inference_fn(model, testloader, DEVICE)
 return oof, predictions<categorify>
submission['birds'].value_counts(normalize=True )
def run_k_fold(NFOLDS, seed_id):
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for fold_id in range(NFOLDS):
oof_, pred_ = run_training(fold_id, seed_id)
 predictions += pred_ / NFOLDS
oof += oof_
return oof, predictions<import_modules>
submission.to_csv("submission.csv", index=False )
from tqdm.notebook import tqdm<categorify>
submission.to_csv("submission.csv", index=False )
<feature_engineering><EOS> 
class StrongBCEwithLogitsMixUp(nn.Module):
def __init__(
 self,
n_classes: int,
use_onehot_target: bool = True,
label_smoothing_coef: float = 0.0,
mixup_p: float = 0.5,
mixup_type: str = 'wave_sum' 
):
super().__init__()
self.n_classes = n_classes
self.use_onehot_target = use_onehot_target
self.label_smoothing_coef = label_smoothing_coef
self.mixup_p = mixup_p
if mixup_type not in ['wave_sum']:
raise ValueError("Wrong mixup_type")
 self.mixup_type = mixup_type
self.loss_f = nn.BCEWithLogitsLoss()
def _sum_mixup(self, inputs, targets):
indices = torch.randperm(inputs.size(0))
shuffled_inputs = inputs[indices]
shuffled_targets = targets[indices]
inputs = inputs + shuffled_inputs
targets = targets + shuffled_targets
targets = targets.clamp(min=0, max=1.0)
 for i in range(inputs.shape[0]):
inputs[i,:] = inputs[i,:] / inputs[i,:].abs().max()
return inputs, targets
def forward(self, batch, model):
wave = batch['waveform']
target = batch['targets']
if self.use_onehot_target:
one_hot_target = torch.nn.functional.one_hot(target, self.n_classes)
 primary_target = target
else:
one_hot_target = target
primary_target = batch['primary_label']
if model.training and np.random.binomial(n=1,p=self.mixup_p):
if self.mixup_type == 'wave_sum':
wave, one_hot_target = self._sum_mixup(wave, one_hot_target)
 logits = model(wave)
 if isinstance(logits, dict):
logits = logits['clipwise_output']
if self.label_smoothing_coef > 0 and model.training:
one_hot_target = torch.abs(one_hot_target.float() - self.label_smoothing_coef)
 else:
one_hot_target = one_hot_target.float()
loss = self.loss_f(logits, one_hot_target)
 losses = {
"loss":loss
}
outputs = {
"logits":logits
}
inputs = {
"targets":primary_target,
"all_targets": target if not self.use_onehot_target else None
}
return losses, inputs, outputs