Kaggle competition: Costa Rican Household Poverty Level Prediction
<SOS> metric: MacroFScore Kaggle data source: costa-rican-household-poverty-level-prediction<compute_test_metric>
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
print(submodel.evaluate(x_train, y_train))
print(submodel.evaluate(x_test, y_test))<import_modules>
pd.options.display.max_columns = None
from pathlib import Path
import pandas as pd
import torch
from fastprogress import progress_bar
import numpy as np
import warnings
from collections import defaultdict
from collections import Counter<set_options>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 train.head()
torch.manual_seed(42)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
np.random.seed(42 )<load_from_csv>
mapping = {"yes": 1, "no": 0}
for df in [train, test]:
df['dependency'] = df['dependency'].replace(mapping ).astype(np.float64)
 df['edjefa'] = df['edjefa'].replace(mapping ).astype(np.float64)
 df['edjefe'] = df['edjefe'].replace(mapping ).astype(np.float64)
 train[['dependency', 'edjefa', 'edjefe']].describe()
ROOT = Path.cwd().parent
INPUT_ROOT = ROOT / "input"
RAW_DATA = INPUT_ROOT / "birdsong-recognition"
TRAIN_AUDIO_DIR = RAW_DATA / "train_audio"
TEST_AUDIO_DIR = RAW_DATA / "test_audio"
if not TEST_AUDIO_DIR.exists() :
TEST_AUDIO_DIR = INPUT_ROOT / "birdcall-check" / "test_audio"
test = pd.read_csv(INPUT_ROOT / "birdcall-check" / "test.csv")
 else:
test = pd.read_csv(RAW_DATA / "test.csv")
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu" )<define_variables>
test['Target'] = np.nan
data = train.append(test, ignore_index = True )
BIRD_CODE = {
'aldfly': 0, 'ameavo': 1, 'amebit': 2, 'amecro': 3, 'amegfi': 4,
'amekes': 5, 'amepip': 6, 'amered': 7, 'amerob': 8, 'amewig': 9,
'amewoo': 10, 'amtspa': 11, 'annhum': 12, 'astfly': 13, 'baisan': 14,
'baleag': 15, 'balori': 16, 'banswa': 17, 'barswa': 18, 'bawwar': 19,
'belkin1': 20, 'belspa2': 21, 'bewwre': 22, 'bkbcuc': 23, 'bkbmag1': 24,
'bkbwar': 25, 'bkcchi': 26, 'bkchum': 27, 'bkhgro': 28, 'bkpwar': 29,
'bktspa': 30, 'blkpho': 31, 'blugrb1': 32, 'blujay': 33, 'bnhcow': 34,
'boboli': 35, 'bongul': 36, 'brdowl': 37, 'brebla': 38, 'brespa': 39,
'brncre': 40, 'brnthr': 41, 'brthum': 42, 'brwhaw': 43, 'btbwar': 44,
'btnwar': 45, 'btywar': 46, 'buffle': 47, 'buggna': 48, 'buhvir': 49,
'bulori': 50, 'bushti': 51, 'buwtea': 52, 'buwwar': 53, 'cacwre': 54,
'calgul': 55, 'calqua': 56, 'camwar': 57, 'cangoo': 58, 'canwar': 59,
'canwre': 60, 'carwre': 61, 'casfin': 62, 'caster1': 63, 'casvir': 64,
'cedwax': 65, 'chispa': 66, 'chiswi': 67, 'chswar': 68, 'chukar': 69,
'clanut': 70, 'cliswa': 71, 'comgol': 72, 'comgra': 73, 'comloo': 74,
'commer': 75, 'comnig': 76, 'comrav': 77, 'comred': 78, 'comter': 79,
'comyel': 80, 'coohaw': 81, 'coshum': 82, 'cowscj1': 83, 'daejun': 84,
'doccor': 85, 'dowwoo': 86, 'dusfly': 87, 'eargre': 88, 'easblu': 89,
'easkin': 90, 'easmea': 91, 'easpho': 92, 'eastow': 93, 'eawpew': 94,
'eucdov': 95, 'eursta': 96, 'evegro': 97, 'fiespa': 98, 'fiscro': 99,
'foxspa': 100, 'gadwal': 101, 'gcrfin': 102, 'gnttow': 103, 'gnwtea': 104,
'gockin': 105, 'gocspa': 106, 'goleag': 107, 'grbher3': 108, 'grcfly': 109,
'greegr': 110, 'greroa': 111, 'greyel': 112, 'grhowl': 113, 'grnher': 114,
'grtgra': 115, 'grycat': 116, 'gryfly': 117, 'haiwoo': 118, 'hamfly': 119,
'hergul': 120, 'herthr': 121, 'hoomer': 122, 'hoowar': 123, 'horgre': 124,
'horlar': 125, 'houfin': 126, 'houspa': 127, 'houwre': 128, 'indbun': 129,
'juntit1': 130, 'killde': 131, 'labwoo': 132, 'larspa': 133, 'lazbun': 134,
'leabit': 135, 'leafly': 136, 'leasan': 137, 'lecthr': 138, 'lesgol': 139,
'lesnig': 140, 'lesyel': 141, 'lewwoo': 142, 'linspa': 143, 'lobcur': 144,
'lobdow': 145, 'logshr': 146, 'lotduc': 147, 'louwat': 148, 'macwar': 149,
'magwar': 150, 'mallar3': 151, 'marwre': 152, 'merlin': 153, 'moublu': 154,
'mouchi': 155, 'moudov': 156, 'norcar': 157, 'norfli': 158, 'norhar2': 159,
'normoc': 160, 'norpar': 161, 'norpin': 162, 'norsho': 163, 'norwat': 164,
'nrwswa': 165, 'nutwoo': 166, 'olsfly': 167, 'orcwar': 168, 'osprey': 169,
'ovenbi1': 170, 'palwar': 171, 'pasfly': 172, 'pecsan': 173, 'perfal': 174,
'phaino': 175, 'pibgre': 176, 'pilwoo': 177, 'pingro': 178, 'pinjay': 179,
'pinsis': 180, 'pinwar': 181, 'plsvir': 182, 'prawar': 183, 'purfin': 184,
'pygnut': 185, 'rebmer': 186, 'rebnut': 187, 'rebsap': 188, 'rebwoo': 189,
'redcro': 190, 'redhea': 191, 'reevir1': 192, 'renpha': 193, 'reshaw': 194,
'rethaw': 195, 'rewbla': 196, 'ribgul': 197, 'rinduc': 198, 'robgro': 199,
'rocpig': 200, 'rocwre': 201, 'rthhum': 202, 'ruckin': 203, 'rudduc': 204,
'rufgro': 205, 'rufhum': 206, 'rusbla': 207, 'sagspa1': 208, 'sagthr': 209,
'savspa': 210, 'saypho': 211, 'scatan': 212, 'scoori': 213, 'semplo': 214,
'semsan': 215, 'sheowl': 216, 'shshaw': 217, 'snobun': 218, 'snogoo': 219,
'solsan': 220, 'sonspa': 221, 'sora': 222, 'sposan': 223, 'spotow': 224,
'stejay': 225, 'swahaw': 226, 'swaspa': 227, 'swathr': 228, 'treswa': 229,
'truswa': 230, 'tuftit': 231, 'tunswa': 232, 'veery': 233, 'vesspa': 234,
'vigswa': 235, 'warvir': 236, 'wesblu': 237, 'wesgre': 238, 'weskin': 239,
'wesmea': 240, 'wessan': 241, 'westan': 242, 'wewpew': 243, 'whbnut': 244,
'whcspa': 245, 'whfibi': 246, 'whtspa': 247, 'whtswi': 248, 'wilfly': 249,
'wilsni1': 250, 'wiltur': 251, 'winwre3': 252, 'wlswar': 253, 'wooduc': 254,
'wooscj2': 255, 'woothr': 256, 'y00475': 257, 'yebfly': 258, 'yebsap': 259,
'yehbla': 260, 'yelwar': 261, 'yerwar': 262, 'yetvir': 263
}
INV_BIRD_CODE = {v: k for k, v in BIRD_CODE.items() }<import_modules>
all_equal = train.groupby('idhogar')['Target'].apply(lambda x: x.nunique() == 1)
 not_equal = all_equal[all_equal != True]
print('There are {} households where the family members do not all have the same target.'.format(len(not_equal)) )
class DFTBase(nn.Module):
def __init__(self):

 super(DFTBase, self ).__init__()
def dft_matrix(self, n): 
(x, y)= np.meshgrid(np.arange(n), np.arange(n))
omega = np.exp(-2 * np.pi * 1j / n)
 W = np.power(omega, x * y)
 return W
def idft_matrix(self, n): 
(x, y)= np.meshgrid(np.arange(n), np.arange(n))
omega = np.exp(2 * np.pi * 1j / n)
 W = np.power(omega, x * y)
 return W
class STFT(DFTBase):
def __init__(self, n_fft=2048, hop_length=None, win_length=None,
window='hann', center=True, pad_mode='reflect', freeze_parameters=True):

 super(STFT, self ).__init__()
assert pad_mode in ['constant', 'reflect']
self.n_fft = n_fft
self.center = center
self.pad_mode = pad_mode
if win_length is None:
win_length = n_fft
if hop_length is None:
hop_length = int(win_length // 4)
 fft_window = librosa.filters.get_window(window, win_length, fftbins=True)
 fft_window = librosa.util.pad_center(fft_window, n_fft)
 self.W = self.dft_matrix(n_fft)
 out_channels = n_fft // 2 + 1
self.conv_real = nn.Conv1d(in_channels=1, out_channels=out_channels,
kernel_size=n_fft, stride=hop_length, padding=0, dilation=1,
groups=1, bias=False)
 self.conv_imag = nn.Conv1d(in_channels=1, out_channels=out_channels,
kernel_size=n_fft, stride=hop_length, padding=0, dilation=1,
groups=1, bias=False)
 self.conv_real.weight.data = torch.Tensor(
 np.real(self.W[:, 0 : out_channels] * fft_window[:, None] ).T)[:, None, :]
self.conv_imag.weight.data = torch.Tensor(
 np.imag(self.W[:, 0 : out_channels] * fft_window[:, None] ).T)[:, None, :]
if freeze_parameters:
for param in self.parameters() :
param.requires_grad = False
def forward(self, input):

 x = input[:, None, :]
if self.center:
x = F.pad(x, pad=(self.n_fft // 2, self.n_fft // 2), mode=self.pad_mode)
 real = self.conv_real(x)
 imag = self.conv_imag(x)
 real = real[:, None, :, :].transpose(2, 3)
 imag = imag[:, None, :, :].transpose(2, 3)
 return real, imag
class Spectrogram(nn.Module):
def __init__(self, n_fft=2048, hop_length=None, win_length=None,
window='hann', center=True, pad_mode='reflect', power=2.0,
freeze_parameters=True):

 super(Spectrogram, self ).__init__()
self.power = power
self.stft = STFT(n_fft=n_fft, hop_length=hop_length,
win_length=win_length, window=window, center=center,
pad_mode=pad_mode, freeze_parameters=True)
 def forward(self, input):

(real, imag)= self.stft.forward(input)
 spectrogram = real ** 2 + imag ** 2
if self.power == 2.0:
pass
else:
spectrogram = spectrogram **(power / 2.0)
 return spectrogram
class LogmelFilterBank(nn.Module):
def __init__(self, sr=32000, n_fft=2048, n_mels=64, fmin=50, fmax=14000, is_log=True,
ref=1.0, amin=1e-10, top_db=80.0, freeze_parameters=True):

 super(LogmelFilterBank, self ).__init__()
self.is_log = is_log
self.ref = ref
self.amin = amin
self.top_db = top_db
self.melW = librosa.filters.mel(sr=sr, n_fft=n_fft, n_mels=n_mels,
fmin=fmin, fmax=fmax ).T
self.melW = nn.Parameter(torch.Tensor(self.melW))
if freeze_parameters:
for param in self.parameters() :
param.requires_grad = False
def forward(self, input):

 mel_spectrogram = torch.matmul(input, self.melW)
 if self.is_log:
output = self.power_to_db(mel_spectrogram)
 else:
output = mel_spectrogram
return output
def power_to_db(self, input):

 ref_value = self.ref
log_spec = 10.0 * torch.log10(torch.clamp(input, min=self.amin, max=np.inf))
log_spec -= 10.0 * np.log10(np.maximum(self.amin, ref_value))
if self.top_db is not None:
if self.top_db < 0:
raise ParameterError('top_db must be non-negative')
 log_spec = torch.clamp(log_spec, min=log_spec.max().item() - self.top_db, max=np.inf)
 return log_spec
class DropStripes(nn.Module):
def __init__(self, dim, drop_width, stripes_num):

 super(DropStripes, self ).__init__()
assert dim in [2, 3]
self.dim = dim
self.drop_width = drop_width
self.stripes_num = stripes_num
def forward(self, input):

 assert input.ndimension() == 4
if self.training is False:
return input
else:
batch_size = input.shape[0]
total_width = input.shape[self.dim]
for n in range(batch_size):
self.transform_slice(input[n], total_width)
 return input
def transform_slice(self, e, total_width):

 for _ in range(self.stripes_num):
distance = torch.randint(low=0, high=self.drop_width, size=(1,)) [0]
bgn = torch.randint(low=0, high=total_width - distance, size=(1,)) [0]
if self.dim == 2:
e[:, bgn : bgn + distance, :] = 0
elif self.dim == 3:
e[:, :, bgn : bgn + distance] = 0
class SpecAugmentation(nn.Module):
def __init__(self, time_drop_width, time_stripes_num, freq_drop_width,
freq_stripes_num):

 super(SpecAugmentation, self ).__init__()
self.time_dropper = DropStripes(dim=2, drop_width=time_drop_width,
stripes_num=time_stripes_num)
 self.freq_dropper = DropStripes(dim=3, drop_width=freq_drop_width,
stripes_num=freq_stripes_num)
 def forward(self, input):
x = self.time_dropper(input)
 x = self.freq_dropper(x)
 return x<load_pretrained>
train[train['idhogar'] == not_equal.index[0]][['idhogar', 'parentesco1', 'Target']]
def get_model(ModelClass: object, config: dict, weights_path: str):
model = ModelClass(**config)
 checkpoint = torch.load(weights_path, map_location='cpu')
 model.load_state_dict(checkpoint["model"])
 model.to(device)
 model.eval()
return model<define_variables>
households_leader = train.groupby('idhogar')['parentesco1'].sum()
households_no_head = train.loc[train['idhogar'].isin(households_leader[households_leader == 0].index), :]
print('There are {} households without a head.'.format(households_no_head['idhogar'].nunique()))
list_of_models = [
{
"model_class": PANNsDense121Att,
"config": {
"sample_rate": 32000,
"window_size": 1024,
"hop_size": 320,
"mel_bins": 64,
"fmin": 50,
"fmax": 14000,
"classes_num": 264,
"apply_aug": True,
"top_db": None
},
"weights_path": ".. /input/birdsongdetectionfinalsubmission1/final_sed_dense121_nomix_fold0_checkpoint_50_score0.7057.pt",
"clip_threshold": 0.3,
"threshold": 0.3
},
{
"model_class": PANNsDense121Att,
"config": {
"sample_rate": 32000,
"window_size": 1024,
"hop_size": 320,
"mel_bins": 64,
"fmin": 50,
"fmax": 14000,
"classes_num": 264,
"apply_aug": True,
"top_db": None
},
"weights_path": ".. /input/birdsongdetectionfinalsubmission1/final_sed_dense121_nomix_fold1_checkpoint_48_score0.6943.pt",
"clip_threshold": 0.3,
"threshold": 0.3
},
{
"model_class": PANNsDense121Att,
"config": {
"sample_rate": 32000,
"window_size": 1024,
"hop_size": 320,
"mel_bins": 64,
"fmin": 50,
"fmax": 14000,
"classes_num": 264,
"apply_aug": True,
"top_db": None
},
"weights_path": ".. /input/birdsongdetectionfinalsubmission1/final_sed_dense121_nomix_fold2_augd_checkpoint_50_score0.6666.pt",
"clip_threshold": 0.3,
"threshold": 0.3
},
{
"model_class": PANNsDense121Att,
"config": {
"sample_rate": 32000,
"window_size": 1024,
"hop_size": 320,
"mel_bins": 64,
"fmin": 50,
"fmax": 14000,
"classes_num": 264,
"apply_aug": True,
"top_db": None
},
"weights_path": ".. /input/birdsongdetectionfinalsubmission1/final_sed_dense121_nomix_fold3_augd_checkpoint_50_score0.6713.pt",
"clip_threshold": 0.3,
"threshold": 0.3
},
{
"model_class": PANNsDense121Att,
"config": {
"sample_rate": 32000,
"window_size": 1024,
"hop_size": 320,
"mel_bins": 64,
"fmin": 50,
"fmax": 14000,
"classes_num": 264,
"apply_aug": True,
"top_db": None
},
"weights_path": ".. /input/birdsongdetectionfinalsubmission1/final_5fold_sed_dense121_nomix_fold0_checkpoint_50_score0.7219.pt",
"clip_threshold": 0.3,
"threshold": 0.3
},
{
"model_class": PANNsDense121Att,
"config": {
"sample_rate": 32000,
"window_size": 1024,
"hop_size": 320,
"mel_bins": 64,
"fmin": 50,
"fmax": 14000,
"classes_num": 264,
"apply_aug": True,
"top_db": None
},
"weights_path": ".. /input/birdsongdetectionfinalsubmission1/final_5fold_sed_dense121_nomix_fold1_checkpoint_44_score0.7645.pt",
"clip_threshold": 0.3,
"threshold": 0.3
},
{
"model_class": PANNsDense121Att,
"config": {
"sample_rate": 32000,
"window_size": 1024,
"hop_size": 320,
"mel_bins": 64,
"fmin": 50,
"fmax": 14000,
"classes_num": 264,
"apply_aug": True,
"top_db": None
},
"weights_path": ".. /input/birdsongdetectionfinalsubmission1/final_5fold_sed_dense121_nomix_fold2_checkpoint_50_score0.7737.pt",
"clip_threshold": 0.3,
"threshold": 0.3
},
{
"model_class": PANNsDense121Att,
"config": {
"sample_rate": 32000,
"window_size": 1024,
"hop_size": 320,
"mel_bins": 64,
"fmin": 50,
"fmax": 14000,
"classes_num": 264,
"apply_aug": True,
"top_db": None
},
"weights_path": ".. /input/birdsongdetectionfinalsubmission1/final_5fold_sed_dense121_nomix_fold3_checkpoint_48_score0.7746.pt",
"clip_threshold": 0.3,
"threshold": 0.3
},
{
"model_class": PANNsDense121Att,
"config": {
"sample_rate": 32000,
"window_size": 1024,
"hop_size": 320,
"mel_bins": 64,
"fmin": 50,
"fmax": 14000,
"classes_num": 264,
"apply_aug": True,
"top_db": None
},
"weights_path": ".. /input/birdsongdetectionfinalsubmission1/final_5fold_sed_dense121_nomix_fold4_checkpoint_50_score0.7728.pt",
"clip_threshold": 0.3,
"threshold": 0.3
},
{
"model_class": PANNsDense121Att,
"config": {
"sample_rate": 32000,
"window_size": 1024,
"hop_size": 320,
"mel_bins": 64,
"fmin": 50,
"fmax": 14000,
"classes_num": 264,
"apply_aug": True,
"top_db": None
},
"weights_path": ".. /input/birdsongdetectionfinalsubmission1/final_sed_dense121_mix_fold0_2_checkpoint_50_score0.6842.pt",
"clip_threshold": 0.3,
"threshold": 0.3
},
{
"model_class": PANNsDense121Att,
"config": {
"sample_rate": 32000,
"window_size": 1024,
"hop_size": 320,
"mel_bins": 64,
"fmin": 50,
"fmax": 14000,
"classes_num": 264,
"apply_aug": True,
"top_db": None
},
"weights_path": ".. /input/birdsongdetectionfinalsubmission1/final_sed_dense121_mix_fold1_2_checkpoint_50_score0.6629.pt",
"clip_threshold": 0.3,
"threshold": 0.3
},
{
"model_class": PANNsDense121Att,
"config": {
"sample_rate": 32000,
"window_size": 1024,
"hop_size": 320,
"mel_bins": 64,
"fmin": 50,
"fmax": 14000,
"classes_num": 264,
"apply_aug": True,
"top_db": None
},
"weights_path": ".. /input/birdsongdetectionfinalsubmission1/final_sed_dense121_mix_fold2_2_checkpoint_50_score0.6884.pt",
"clip_threshold": 0.3,
"threshold": 0.3
},
{
"model_class": PANNsDense121Att,
"config": {
"sample_rate": 32000,
"window_size": 1024,
"hop_size": 320,
"mel_bins": 64,
"fmin": 50,
"fmax": 14000,
"classes_num": 264,
"apply_aug": True,
"top_db": None
},
"weights_path": ".. /input/birdsongdetectionfinalsubmission1/final_sed_dense121_mix_fold3_2_checkpoint_50_score0.6870.pt",
"clip_threshold": 0.3,
"threshold": 0.3
}
]
PERIOD = 30
SR = 32000
vote_lim = 4
TTA = 10<feature_engineering>
households_no_head_equal = households_no_head.groupby('idhogar')['Target'].apply(lambda x: x.nunique() == 1)
 print('{} Households with no head have different labels.'.format(sum(households_no_head_equal == False)) )
for lm in list_of_models:
lm["model"] = get_model(lm["model_class"], lm["config"], lm["weights_path"] )<categorify>
for household in not_equal.index:
true_target = int(train[(train['idhogar'] == household)&(train['parentesco1'] == 1.0)]['Target'])
 train.loc[train['idhogar'] == household, 'Target'] = true_target
all_equal = train.groupby('idhogar')['Target'].apply(lambda x: x.nunique() == 1)
 not_equal = all_equal[all_equal != True]
print('There are {} households where the family members do not all have the same target.'.format(len(not_equal)) )
def prediction_for_clip(test_df: pd.DataFrame,
clip: np.ndarray,
model,
threshold,
clip_threshold):
audios = []
y = clip.astype(np.float32)
 len_y = len(y)
 start = 0
end = PERIOD * SR
while True:
y_batch = y[start:end].astype(np.float32)
 if len(y_batch)!= PERIOD * SR:
y_pad = np.zeros(PERIOD * SR, dtype=np.float32)
 y_pad[:len(y_batch)] = y_batch
audios.append(y_pad)
 break
start = end
end += PERIOD * SR
audios.append(y_batch)
 array = np.asarray(audios)
 tensors = torch.from_numpy(array)
 model.eval()
estimated_event_list = []
global_time = 0.0
site = test_df["site"].values[0]
audio_id = test_df["audio_id"].values[0]
for image in tensors:
image = image.unsqueeze(0 ).unsqueeze(0)
 image = image.expand(image.shape[0], TTA, image.shape[2])
 image = image.to(device)
 with torch.no_grad() :
prediction = model(( image, None))
framewise_outputs = prediction["framewise_output"].detach(
 ).cpu().numpy() [0].mean(axis=0)
 clipwise_outputs = prediction["clipwise_output"].detach(
 ).cpu().numpy() [0].mean(axis=0)
 thresholded = framewise_outputs >= threshold
clip_thresholded = clipwise_outputs >= clip_threshold
clip_indices = np.argwhere(clip_thresholded ).reshape(-1)
 clip_codes = []
for ci in clip_indices:
clip_codes.append(INV_BIRD_CODE[ci])
 for target_idx in range(thresholded.shape[1]):
if thresholded[:, target_idx].mean() == 0:
pass
else:
detected = np.argwhere(thresholded[:, target_idx] ).reshape(-1)
 head_idx = 0
tail_idx = 0
while True:
if(tail_idx + 1 == len(detected)) or(
 detected[tail_idx + 1] -
detected[tail_idx] != 1):
onset = 0.01 * detected[
head_idx] + global_time
offset = 0.01 * detected[
tail_idx] + global_time
onset_idx = detected[head_idx]
offset_idx = detected[tail_idx]
max_confidence = framewise_outputs[
onset_idx:offset_idx, target_idx].max()
mean_confidence = framewise_outputs[
onset_idx:offset_idx, target_idx].mean()
if INV_BIRD_CODE[target_idx] in clip_codes:
estimated_event = {
"site": site,
"audio_id": audio_id,
"ebird_code": INV_BIRD_CODE[target_idx],
"clip_codes": clip_codes,
"onset": onset,
"offset": offset,
"max_confidence": max_confidence,
"mean_confidence": mean_confidence
}
estimated_event_list.append(estimated_event)
 head_idx = tail_idx + 1
tail_idx = tail_idx + 1
if head_idx >= len(detected):
break
else:
tail_idx += 1
global_time += PERIOD
prediction_df = pd.DataFrame(estimated_event_list)
 return prediction_df
def prediction(test_df: pd.DataFrame,
test_audio: Path,
list_of_model_details):
unique_audio_id = test_df.audio_id.unique()
warnings.filterwarnings("ignore")
 prediction_dfs_dict = defaultdict(list)
 for audio_id in progress_bar(unique_audio_id):
clip, _ = librosa.load(test_audio /(audio_id + ".mp3"),
sr=SR,
mono=True,
res_type="kaiser_fast")
 test_df_for_audio_id = test_df.query(
 f"audio_id == '{audio_id}'" ).reset_index(drop=True)
 for i, model_details in enumerate(list_of_model_details):
prediction_df = prediction_for_clip(test_df_for_audio_id,
clip=clip,
model=model_details["model"],
threshold=model_details["threshold"],
clip_threshold=model_details["clip_threshold"])
 prediction_dfs_dict[i].append(prediction_df)
 list_of_prediction_df = []
for key, prediction_dfs in prediction_dfs_dict.items() :
prediction_df = pd.concat(prediction_dfs, axis=0, sort=False ).reset_index(drop=True)
 list_of_prediction_df.append(prediction_df)
 return list_of_prediction_df<predict_on_test>
missing = pd.DataFrame(data.isnull().sum() ).rename(columns = {0: 'total'})
 missing['percent'] = missing['total'] / len(data)
 missing.sort_values('percent', ascending = False ).head(10 ).drop('Target' )
list_of_prediction_df = prediction(test_df=test,
test_audio=TEST_AUDIO_DIR,
list_of_model_details=list_of_models )<categorify>
heads.groupby('v18q')['v18q1'].apply(lambda x: x.isnull().sum() )
all_row_id = test[["row_id"]]
list_of_submissions = []
for prediction_df in list_of_prediction_df:
post_processed = get_post_post_process_predictions(prediction_df)
 submission = post_processed.fillna("nocall")
 submission = submission.set_index('row_id')
 list_of_submissions.append(submission )<define_variables>
data['v18q1'] = data['v18q1'].fillna(0 )
list_all_of_row_ids = []
for sub_x in list_of_submissions:
list_all_of_row_ids+= list(sub_x.index.values)
 list_all_of_row_ids = list(set(list_all_of_row_ids))<string_transform>
data.loc[(data['tipovivi1'] == 1), 'v2a1'] = 0
data['v2a1-missing'] = data['v2a1'].isnull()
data['v2a1-missing'].value_counts()
final_submission = []
for row_id in list_all_of_row_ids:
birds = []
for sub in list_of_submissions:
if row_id in sub.index:
birds.extend(sub.loc[row_id].birds.split(" "))
birds = [x for x in birds if "nocall" != x and "" != x]
count_birds = Counter(birds)
 final_birds = []
for key, value in count_birds.items() :
if value >= vote_lim:
final_birds.append(key)
 if len(final_birds)>0:
row_data = {
"row_id": row_id,
"birds": " ".join(sorted(final_birds))
}
else:
row_data = {
"row_id": row_id,
"birds": "nocall"
}
final_submission.append(row_data)
 site_3_data = defaultdict(list)
 for row in final_submission:
if "site_3" in row["row_id"]:
final_row_id = "_".join(row["row_id"].split("_")[0:-1])
 birds = row["birds"].split(" ")
 birds = [x for x in birds if "nocall" != x and "" != x]
site_3_data[final_row_id].extend(birds)
 for key, value in site_3_data.items() :
count_birds = Counter(value)
 final_birds = []
for k, v in count_birds.items() :
if v >= vote_lim:
final_birds.append(k)
 if len(final_birds)>0:
row_data = {
"row_id": key,
"birds": " ".join(sorted(final_birds))
}
else:
row_data = {
"row_id": key,
"birds": "nocall"
}
final_submission.append(row_data)
 final_submission = pd.DataFrame(final_submission)
 final_submission = all_row_id.merge(final_submission, on="row_id", how="left")
 final_submission = final_submission.fillna("nocall" )<save_to_csv>
data.loc[(( data['age'] > 19)|(data['age'] < 7)) &(data['rez_esc'].isnull()), 'rez_esc'] = 0
data['rez_esc-missing'] = data['rez_esc'].isnull()
final_submission.to_csv("submission.csv", index=False)
 final_submission.head(50 )<install_modules>
data.loc[data['rez_esc'] > 5, 'rez_esc'] = 5
!pip install /kaggle/input/torchlibrosa/torchlibrosa-0.0.4-py3-none-any.whl<import_modules>
id_ = ['Id', 'idhogar', 'Target']
from geffnet import tf_efficientnet_b4_ns, tf_efficientnet_b3_ns, tf_efficientnet_b0_ns, tf_efficientnet_b2_ns<set_options>
ind_bool = ['v18q', 'dis', 'male', 'female', 'estadocivil1', 'estadocivil2', 'estadocivil3',
'estadocivil4', 'estadocivil5', 'estadocivil6', 'estadocivil7',
'parentesco1', 'parentesco2', 'parentesco3', 'parentesco4', 'parentesco5',
'parentesco6', 'parentesco7', 'parentesco8', 'parentesco9', 'parentesco10',
'parentesco11', 'parentesco12', 'instlevel1', 'instlevel2', 'instlevel3',
'instlevel4', 'instlevel5', 'instlevel6', 'instlevel7', 'instlevel8',
'instlevel9', 'mobilephone', 'rez_esc-missing']
ind_ordered = ['rez_esc', 'escolari', 'age']
warnings.filterwarnings("ignore")
<set_options>
hh_bool = ['hacdor', 'hacapo', 'v14a', 'refrig', 'paredblolad', 'paredzocalo',
'paredpreb','pisocemento', 'pareddes', 'paredmad',
'paredzinc', 'paredfibras', 'paredother', 'pisomoscer', 'pisoother',
'pisonatur', 'pisonotiene', 'pisomadera',
'techozinc', 'techoentrepiso', 'techocane', 'techootro', 'cielorazo',
'abastaguadentro', 'abastaguafuera', 'abastaguano',
'public', 'planpri', 'noelec', 'coopele', 'sanitario1',
'sanitario2', 'sanitario3', 'sanitario5', 'sanitario6',
'energcocinar1', 'energcocinar2', 'energcocinar3', 'energcocinar4',
'elimbasu1', 'elimbasu2', 'elimbasu3', 'elimbasu4',
'elimbasu5', 'elimbasu6', 'epared1', 'epared2', 'epared3',
'etecho1', 'etecho2', 'etecho3', 'eviv1', 'eviv2', 'eviv3',
'tipovivi1', 'tipovivi2', 'tipovivi3', 'tipovivi4', 'tipovivi5',
'computer', 'television', 'lugar1', 'lugar2', 'lugar3',
'lugar4', 'lugar5', 'lugar6', 'area1', 'area2', 'v2a1-missing']
hh_ordered = [ 'rooms', 'r4h1', 'r4h2', 'r4h3', 'r4m1','r4m2','r4m3', 'r4t1', 'r4t2',
'r4t3', 'v18q1', 'tamhog','tamviv','hhsize','hogar_nin',
'hogar_adul','hogar_mayor','hogar_total', 'bedrooms', 'qmobilephone']
hh_cont = ['v2a1', 'dependency', 'edjefe', 'edjefa', 'meaneduc', 'overcrowding']
def set_seed(seed: int = 42):
random.seed(seed)
 np.random.seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.cuda.manual_seed_all(seed)
 torch.backends.cudnn.benchmark = False
set_seed(1213 )<load_from_csv>
sqr_ = ['SQBescolari', 'SQBage', 'SQBhogar_total', 'SQBedjefe',
'SQBhogar_nin', 'SQBovercrowding', 'SQBdependency', 'SQBmeaned', 'agesq']
ROOT = Path.cwd().parent
INPUT_ROOT = ROOT / "input"
RAW_DATA = INPUT_ROOT / "birdsong-recognition"
TRAIN_AUDIO_DIR = RAW_DATA / "train_audio"
TEST_AUDIO_DIR = RAW_DATA / "test_audio"
if not TEST_AUDIO_DIR.exists() :
TEST_AUDIO_DIR = INPUT_ROOT / "birdcall-check" / "test_audio"
test = pd.read_csv(INPUT_ROOT / "birdcall-check" / "test.csv")
 else:
test = pd.read_csv(RAW_DATA / "test.csv" )<load_from_csv>
x = ind_bool + ind_ordered + id_ + hh_bool + hh_ordered + hh_cont + sqr_
print('There are no repeats: ', np.all(np.array(list(Counter(x ).values())) == 1))
print('We covered every variable: ', len(x)== data.shape[1] )
train = pd.read_csv(RAW_DATA / "train.csv")
 label_encoder = LabelEncoder().fit(train.ebird_code.values )<init_hyperparams>
corr_matrix.loc[corr_matrix['tamhog'].abs() > 0.9, corr_matrix['tamhog'].abs() > 0.9]
class model_config:
ROOT_PATH = ".. /input/train_audio"
num_classes = 264
max_duration = 5
sample_rate = 32000
sigmoid = True
batch_size = 32
num_workers = 4
nmels = 128
melspectrogram_parameters = {
"n_mels": 128,
"fmin": 20,
"fmax": 16000,
"hop_length": 320,
"n_fft": 1024
}
threshold = 0.3
augm_spec_prob = 0.
 res_type = "kaiser_best"<choose_model_class>
heads = heads.drop(columns = ['tamhog', 'hogar_total', 'r4t3'] )
def init_layer(layer):

 nn.init.xavier_uniform_(layer.weight)
 if hasattr(layer, "bias"):
if layer.bias is not None:
layer.bias.data.fill_(0.0)
 def init_bn(bn):

 bn.bias.data.fill_(0.0)
 bn.weight.data.fill_(1.0)
 class LogMel(nn.Module):
def __init__(self, params):
super(LogMel, self ).__init__()
params = Namespace(**params)
 self.spectrogram_extractor = torchlibrosa.stft.Spectrogram(n_fft=params.n_fft, hop_length=params.hop_length)
 self.logmel_extractor = torchlibrosa.stft.LogmelFilterBank(
 sr=32000,
n_fft=params.n_fft,
n_mels=params.n_mels,
top_db=None,
fmin=params.fmin,
fmax=params.fmax,
is_log=False)
 def forward(self, sound):
spec = self.spectrogram_extractor(sound)
 spec = self.logmel_extractor(spec)
 spec = torch.log(1e-8 + spec)
 return spec / 10.
 class AttBlock(nn.Module):
def __init__(self, n_in, n_out, activation="linear", temperature=1.0):
super(AttBlock, self ).__init__()
self.activation = activation
self.temperature = temperature
self.att = nn.Conv1d(
 in_channels=n_in,
out_channels=n_out,
kernel_size=1,
stride=1,
padding=0,
bias=True, 
)
 self.cla = nn.Conv1d(
 in_channels=n_in,
out_channels=n_out,
kernel_size=1,
stride=1,
padding=0,
bias=True, 
)
 self.bn_att = nn.BatchNorm1d(n_out)
 self.init_weights()
def init_weights(self):
init_layer(self.att)
 init_layer(self.cla)
 init_bn(self.bn_att)
 def forward(self, x):
norm_att = torch.softmax(torch.clamp(self.att(x), -10, 10), dim=-1)
 cla = self.nonlinear_transform(self.cla(x))
x = torch.sum(norm_att * cla, dim=2)
 return x, norm_att, cla
def nonlinear_transform(self, x):
if self.activation == "linear":
return x
elif self.activation == "sigmoid":
return torch.sigmoid(x)
 class ConvBlock(nn.Module):
def __init__(self, in_channels, out_channels):
super(ConvBlock, self ).__init__()
self.conv1 = nn.Conv2d(
 in_channels=in_channels,
out_channels=out_channels,
kernel_size=(3, 3),
stride=(1, 1),
padding=(1, 1),
bias=False, 
)
 self.conv2 = nn.Conv2d(
 in_channels=out_channels,
out_channels=out_channels,
kernel_size=(3, 3),
stride=(1, 1),
padding=(1, 1),
bias=False, 
)
 self.bn1 = nn.BatchNorm2d(out_channels)
 self.bn2 = nn.BatchNorm2d(out_channels)
 self.init_weight()
def init_weight(self):
init_layer(self.conv1)
 init_layer(self.conv2)
 init_bn(self.bn1)
 init_bn(self.bn2)
 def forward(self, input, pool_size=(2, 2), pool_type="avg"):
x = input
x = F.relu_(self.bn1(self.conv1(x)))
 x = F.relu_(self.bn2(self.conv2(x)))
 if pool_type == "max":
x = F.max_pool2d(x, kernel_size=pool_size)
 elif pool_type == "avg":
x = F.avg_pool2d(x, kernel_size=pool_size)
 elif pool_type == "avg+max":
x1 = F.avg_pool2d(x, kernel_size=pool_size)
 x2 = F.max_pool2d(x, kernel_size=pool_size)
 x = x1 + x2
else:
raise Exception("Incorrect argument!")
 return x
def interpolate(x, ratio):

(batch_size, time_steps, classes_num)= x.shape
upsampled = x[:, :, None, :].repeat(1, 1, ratio, 1)
 upsampled = upsampled.reshape(batch_size, time_steps * ratio, classes_num)
 return upsampled
def pad_framewise_output(framewise_output, frames_num):

 pad = framewise_output[:, -1:, :].repeat(
 1, frames_num - framewise_output.shape[1], 1 
)

output = torch.cat(( framewise_output, pad), dim=1)

return output
class Cnn14_DecisionLevelAtt(nn.Module):
def __init__(self, classes_num, config):
super(Cnn14_DecisionLevelAtt, self ).__init__()
self.interpolate_ratio = 32
self.bn0 = nn.BatchNorm2d(config.melspectrogram_parameters["n_mels"])
 self.conv_block1 = ConvBlock(in_channels=1, out_channels=64)
 self.conv_block2 = ConvBlock(in_channels=64, out_channels=128)
 self.conv_block3 = ConvBlock(in_channels=128, out_channels=256)
 self.conv_block4 = ConvBlock(in_channels=256, out_channels=512)
 self.conv_block5 = ConvBlock(in_channels=512, out_channels=1024)
 self.conv_block6 = ConvBlock(in_channels=1024, out_channels=2048)
 self.fc1 = nn.Linear(2048, 2048, bias=True)
 self.att_block = AttBlock(2048, classes_num, activation="sigmoid")
 self.logmel = LogMel(config.melspectrogram_parameters)
 self.init_weight()
def init_weight(self):
init_bn(self.bn0)
 init_layer(self.fc1)
 def forward(self, x):
x = self.logmel(x)
 frames_num = x.shape[2]
x = x.transpose(1, 3)
 x = self.bn0(x)
 x = x.transpose(1, 3)
 x = self.conv_block1(x, pool_size=(2, 2), pool_type="avg")
 x = F.dropout(x, p=0.2, training=self.training)
 x = self.conv_block2(x, pool_size=(2, 2), pool_type="avg")
 x = F.dropout(x, p=0.2, training=self.training)
 x = self.conv_block3(x, pool_size=(2, 2), pool_type="avg")
 x = F.dropout(x, p=0.2, training=self.training)
 x = self.conv_block4(x, pool_size=(2, 2), pool_type="avg")
 x = F.dropout(x, p=0.2, training=self.training)
 x = self.conv_block5(x, pool_size=(2, 2), pool_type="avg")
 x = F.dropout(x, p=0.2, training=self.training)
 x = self.conv_block6(x, pool_size=(1, 1), pool_type="avg")
 x = F.dropout(x, p=0.2, training=self.training)
 x = torch.mean(x, dim=3)
 x1 = F.max_pool1d(x, kernel_size=3, stride=1, padding=1)
 x2 = F.avg_pool1d(x, kernel_size=3, stride=1, padding=1)
 x = x1 + x2
x = F.dropout(x, p=0.5, training=self.training)
 x = x.transpose(1, 2)
 x = F.relu_(self.fc1(x))
x = x.transpose(1, 2)
 x = F.dropout(x, p=0.5, training=self.training)
(clipwise_output, _, segmentwise_output)= self.att_block(x)
 return clipwise_output<concatenate>
elec = []
for i, row in heads.iterrows() :
if row['noelec'] == 1:
elec.append(0)
 elif row['coopele'] == 1:
elec.append(1)
 elif row['public'] == 1:
elec.append(2)
 elif row['planpri'] == 1:
elec.append(3)
 else:
elec.append(np.nan)
 heads['elec'] = elec
heads['elec-missing'] = heads['elec'].isnull()
def _resnet_conv3x3(in_planes, out_planes):
return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=1,
padding=1, groups=1, bias=False, dilation=1)
 def _resnet_conv1x1(in_planes, out_planes):
return nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=1, bias=False)
 class _ResnetBasicBlock(nn.Module):
expansion = 1
def __init__(self, inplanes, planes, stride=1, downsample=None, groups=1,
base_width=64, dilation=1, norm_layer=None):
super(_ResnetBasicBlock, self ).__init__()
if norm_layer is None:
norm_layer = nn.BatchNorm2d
if groups != 1 or base_width != 64:
raise ValueError('_ResnetBasicBlock only supports groups=1 and base_width=64')
 if dilation > 1:
raise NotImplementedError("Dilation > 1 not supported in _ResnetBasicBlock")
 self.stride = stride
self.conv1 = _resnet_conv3x3(inplanes, planes)
 self.bn1 = norm_layer(planes)
 self.relu = nn.ReLU(inplace=True)
 self.conv2 = _resnet_conv3x3(planes, planes)
 self.bn2 = norm_layer(planes)
 self.downsample = downsample
self.stride = stride
self.init_weights()
def init_weights(self):
init_layer(self.conv1)
 init_bn(self.bn1)
 init_layer(self.conv2)
 init_bn(self.bn2)
 nn.init.constant_(self.bn2.weight, 0)
 def forward(self, x):
identity = x
if self.stride == 2:
out = F.avg_pool2d(x, kernel_size=(2, 2))
else:
out = x
out = self.conv1(out)
 out = self.bn1(out)
 out = self.relu(out)
 out = F.dropout(out, p=0.1, training=self.training)
 out = self.conv2(out)
 out = self.bn2(out)
 if self.downsample is not None:
identity = self.downsample(identity)
 out += identity
out = self.relu(out)
 return out
class _ResnetBottleneck(nn.Module):
expansion = 4
def __init__(self, inplanes, planes, stride=1, downsample=None, groups=1,
base_width=64, dilation=1, norm_layer=None):
super(_ResnetBottleneck, self ).__init__()
if norm_layer is None:
norm_layer = nn.BatchNorm2d
width = int(planes *(base_width / 64.))* groups
self.stride = stride
self.conv1 = _resnet_conv1x1(inplanes, width)
 self.bn1 = norm_layer(width)
 self.conv2 = _resnet_conv3x3(width, width)
 self.bn2 = norm_layer(width)
 self.conv3 = _resnet_conv1x1(width, planes * self.expansion)
 self.bn3 = norm_layer(planes * self.expansion)
 self.relu = nn.ReLU(inplace=True)
 self.downsample = downsample
self.stride = stride
self.init_weights()
def init_weights(self):
init_layer(self.conv1)
 init_bn(self.bn1)
 init_layer(self.conv2)
 init_bn(self.bn2)
 init_layer(self.conv3)
 init_bn(self.bn3)
 nn.init.constant_(self.bn3.weight, 0)
 def forward(self, x):
identity = x
if self.stride == 2:
x = F.avg_pool2d(x, kernel_size=(2, 2))
out = self.conv1(x)
 out = self.bn1(out)
 out = self.relu(out)
 out = self.conv2(out)
 out = self.bn2(out)
 out = self.relu(out)
 out = F.dropout(out, p=0.1, training=self.training)
 out = self.conv3(out)
 out = self.bn3(out)
 if self.downsample is not None:
identity = self.downsample(identity)
 out += identity
out = self.relu(out)
 return out
class _ResNet(nn.Module):
def __init__(self, block, layers, zero_init_residual=False,
groups=1, width_per_group=64, replace_stride_with_dilation=None,
norm_layer=None):
super(_ResNet, self ).__init__()
if norm_layer is None:
norm_layer = nn.BatchNorm2d
self._norm_layer = norm_layer
self.inplanes = 64
self.dilation = 1
if replace_stride_with_dilation is None:
replace_stride_with_dilation = [False, False, False]
if len(replace_stride_with_dilation)!= 3:
raise ValueError("replace_stride_with_dilation should be None "
"or a 3-element tuple, got {}".format(replace_stride_with_dilation))
self.groups = groups
self.base_width = width_per_group
self.layer1 = self._make_layer(block, 64, layers[0], stride=1)
 self.layer2 = self._make_layer(block, 128, layers[1], stride=2,
dilate=replace_stride_with_dilation[0])
 self.layer3 = self._make_layer(block, 256, layers[2], stride=2,
dilate=replace_stride_with_dilation[1])
 self.layer4 = self._make_layer(block, 512, layers[3], stride=2,
dilate=replace_stride_with_dilation[2])
 def _make_layer(self, block, planes, blocks, stride=1, dilate=False):
norm_layer = self._norm_layer
downsample = None
previous_dilation = self.dilation
if dilate:
self.dilation *= stride
stride = 1
if stride != 1 or self.inplanes != planes * block.expansion:
if stride == 1:
downsample = nn.Sequential(
 _resnet_conv1x1(self.inplanes, planes * block.expansion),
norm_layer(planes * block.expansion), 
)
 init_layer(downsample[0])
 init_bn(downsample[1])
 elif stride == 2:
downsample = nn.Sequential(
 nn.AvgPool2d(kernel_size=2),
_resnet_conv1x1(self.inplanes, planes * block.expansion),
norm_layer(planes * block.expansion), 
)
 init_layer(downsample[1])
 init_bn(downsample[2])
 layers = []
layers.append(block(self.inplanes, planes, stride, downsample, self.groups,
self.base_width, previous_dilation, norm_layer))
self.inplanes = planes * block.expansion
for _ in range(1, blocks):
layers.append(block(self.inplanes, planes, groups=self.groups,
base_width=self.base_width, dilation=self.dilation,
norm_layer=norm_layer))
return nn.Sequential(*layers)
 def forward(self, x):
x = self.layer1(x)
 x = self.layer2(x)
 x = self.layer3(x)
 x = self.layer4(x)
 return x
class ResNet38(nn.Module):
def __init__(self, classes_num, config):
super(ResNet38, self ).__init__()
self.bn0 = nn.BatchNorm2d(config.nmels)
 self.conv_block1 = ConvBlock(in_channels=1, out_channels=64)
 self.resnet = _ResNet(block=_ResnetBasicBlock, layers=[3, 4, 6, 3], zero_init_residual=True)
 self.conv_block_after1 = ConvBlock(in_channels=512, out_channels=2048)
 self.fc1 = nn.Linear(2048, 2048)
 self.classifier = nn.Linear(2048, classes_num, bias=True)
 self.init_weights()
self.logmel = LogMel(config.melspectrogram_parameters)
 self.spec_augm = torchlibrosa.augmentation.SpecAugmentation(
 time_drop_width=64,
time_stripes_num=2 *(config.max_duration // 5),
freq_drop_width=8,
freq_stripes_num=2, 
)
 self.spec_augm_prob = config.augm_spec_prob
def init_weights(self):
init_bn(self.bn0)
 init_layer(self.fc1)
 init_layer(self.classifier)
 def forward(self, input):
x = self.logmel(input)
 if self.training:
mask =(torch.rand(x.size(0)) > 0.33 ).to(device)
 x = torch.where(
 torch.repeat_interleave(mask, x.size(2)* x.size(3)).reshape(x.size()),
x, self.spec_augm(x))
x = x.transpose(1, 3)
 x = self.bn0(x)
 x = x.transpose(1, 3)
 x = self.conv_block1(x, pool_size=(2, 2), pool_type='avg')
 x = F.dropout(x, p=0.2, training=self.training, inplace=True)
 x = self.resnet(x)
 x = F.avg_pool2d(x, kernel_size=(2, 2))
x = F.dropout(x, p=0.2, training=self.training, inplace=True)
 x = self.conv_block_after1(x, pool_size=(1, 1), pool_type='avg')
 x = F.dropout(x, p=0.2, training=self.training, inplace=True)
 x = torch.mean(x, dim=3)
(x1, _)= torch.max(x, dim=2)
 x2 = torch.mean(x, dim=2)
 x = x1 + x2
x = F.dropout(x, p=0.5, training=self.training)
 x = F.relu_(self.fc1(x))
embedding = F.dropout(x, p=0.5, training=self.training)
 clipwise_output = self.classifier(x)
 return clipwise_output<load_pretrained>
heads = heads.drop(columns = 'area2')
 heads.groupby('area1')['Target'].value_counts(normalize = True)
def get_model(config):
models = []
sigmoids = []
weights = []
for fold in range(5):
config.model = "b4"
config.vertical_encoding = True
config.batch_norm = True
model = tf_efficientnet_b4_ns(
 config, pretrained=False, num_classes=config.num_classes
).cuda()
state_dict = torch.load(f".. /input/blend5-b4-best/fold_{fold}_test_1_05.pth")
 model.load_state_dict(state_dict, strict=False)
 model.eval()
models.append(model)
 sigmoids.append(True)
 weights.append(1)
 for fold in range(5):
config.model = "b3"
config.vertical_encoding = True
config.batch_norm = True
model = tf_efficientnet_b3_ns(
 config, pretrained=False, num_classes=config.num_classes
).cuda()
state_dict = torch.load(f".. /input/blend5-b3-london/fold_{fold}_test_1_05.pth")
 model.load_state_dict(state_dict, strict=False)
 model.eval()
models.append(model)
 sigmoids.append(True)
 weights.append(1)
 for fold in range(5):
config.model = "b2"
config.vertical_encoding = True
config.batch_norm = True
model = tf_efficientnet_b2_ns(
 config, pretrained=False, num_classes=config.num_classes
).cuda()
state_dict = torch.load(f".. /input/blend5-b2-xbet/fold_{fold}_test_1_05.pth")
 model.load_state_dict(state_dict, strict=False)
 model.eval()
models.append(model)
 sigmoids.append(True)
 weights.append(1)
 for fold in range(5):
config.model = "b0"
config.vertical_encoding = True
config.batch_norm = True
model = tf_efficientnet_b0_ns(
 config, pretrained=False, num_classes=config.num_classes
).cuda()
state_dict = torch.load(f".. /input/blend5-b0-detka/fold_{fold}_test_1_05.pth")
 model.load_state_dict(state_dict, strict=False)
 model.eval()
models.append(model)
 sigmoids.append(True)
 weights.append(1)
 for fold in range(5):
config.model = "cnn14_att"
model = Cnn14_DecisionLevelAtt(config.num_classes, config ).cuda()
state_dict = torch.load(f".. /input/blend5-cnn14-mega/fold_{fold}_test_1_05.pth")
 model.load_state_dict(state_dict, strict=False)
 model.eval()
models.append(model)
 sigmoids.append(False)
 weights.append(2)
 for fold in range(5):
config.model = "r38"
model = ResNet38(config.num_classes, config ).cuda()
state_dict = torch.load(f".. /input/blend5-r38-oxford/fold_{fold}_test_1_05.pth")
 model.load_state_dict(state_dict, strict=False)
 model.eval()
models.append(model)
 sigmoids.append(True)
 weights.append(2)
 return models, sigmoids, weights<import_modules>
heads['roof'] = np.argmax(np.array(heads[['etecho1', 'etecho2', 'etecho3']]),
axis = 1)
 heads = heads.drop(columns = ['etecho1', 'etecho2', 'etecho3'])
 heads['floor'] = np.argmax(np.array(heads[['eviv1', 'eviv2', 'eviv3']]),
axis = 1)
 heads = heads.drop(columns = ['eviv1', 'eviv2', 'eviv3'] )
from librosa.core.audio import __audioread_load as audioread_load<categorify>
heads['warning'] = 1 *(heads['sanitario1'] + 
(heads['elec'] == 0)+
heads['pisonotiene'] +
heads['abastaguano'] + 
(heads['cielorazo'] == 0))
class TestItem:
def __init__(self, audio_id, site, df, spectrograms):
self.audio_id = audio_id
self.site = site
self.df = df
self.spectrograms = torch.from_numpy(spectrograms)
 def pin_memory(self):
self.spectrograms = self.spectrograms.pin_memory()
return self
def __len__(self):
return len(self.df)
 class TestDataset(data.Dataset):
def __init__(self, test, config):
self.test = test
self.config = config
self.unique_audio_id = test.audio_id.unique()
def __len__(self):
return len(self.unique_audio_id)
 def _load_audio(self, filename):
clip, _ = librosa.load(
 TEST_AUDIO_DIR /(filename + ".mp3"), sr=model_config.sample_rate, mono=True 
)
 return clip.astype(np.float32)
 def _load_audio_faster(self, filename):
path = TEST_AUDIO_DIR /(filename + '.mp3')
 clip, sr_native = audioread_load(path, offset=0.0, duration=None, dtype=np.float32)
 clip = librosa.to_mono(clip)
 if sr_native > 0:
clip = librosa.resample(clip, sr_native, model_config.sample_rate, res_type=model_config.res_type)
 return clip.astype(np.float32)
 def _transform(self, audio, max_chunks):
audio = audio[:self.config.sample_rate * self.config.max_duration * max_chunks]
return audio.reshape(max_chunks, self.config.sample_rate * self.config.max_duration)
 def __getitem__(self, idx: int):
audio_id = self.unique_audio_id[idx]
test_df_for_audio_id = test.query(f"audio_id == '{audio_id}'" ).reset_index(
 drop=True 
)
 site = test_df_for_audio_id.iloc[0].site
audio = self._load_audio_faster(audio_id)
 if site != "site_3":
specs = self._transform(audio, len(test_df_for_audio_id))
else:
num_chunks =(len(audio)/ self.config.sample_rate)// 5
specs = self._transform(audio, int(num_chunks))
return TestItem(audio_id, site, test_df_for_audio_id, specs)
 def mo_(output):
if type(output)== tuple:
return output[0]
else:
return output
def prediction_for_clip(model, spectrograms, config):
model.eval()
with torch.no_grad() :
predictions = []
batches = math.ceil(spectrograms.shape[0] / config.batch_size)
 for i in range(batches):
batch = spectrograms[i * config.batch_size :(i + 1)* config.batch_size]
if batch.ndim == 3:
batch = batch.unsqueeze(0)
 pred = mo_(model(batch))
predictions.append(pred)
 predictions = torch.cat(predictions, dim=0)
 if config.sigmoid:
predictions = predictions.sigmoid()
return predictions<find_best_params>
heads['phones-per-capita'] = heads['qmobilephone'] / heads['tamviv']
heads['tablets-per-capita'] = heads['v18q1'] / heads['tamviv']
heads['rooms-per-capita'] = heads['rooms'] / heads['tamviv']
heads['rent-per-capita'] = heads['v2a1'] / heads['tamviv']
models, sigmoids, weights = get_model(model_config )<categorify>
ind = ind.drop(columns = 'male' )
prediction_dict = {}
for i, row_id in enumerate(test_items_top):
events =(predictions_top[i] >= model_config.threshold)
 labels = np.argwhere(events ).reshape(-1 ).tolist()
if len(labels)== 0:
prediction_dict[row_id] = "nocall"
else:
labels_str_list = label_encoder.inverse_transform(labels)
 label_string = " ".join(labels_str_list)
 prediction_dict[row_id] = label_string<save_to_csv>
range_ = lambda x: x.max() - x.min()
range_.__name__ = 'range_'
ind_agg = ind.drop(columns = 'Target' ).groupby('idhogar' ).agg(['min', 'max', 'sum', 'count', 'std', range_])
 ind_agg.head()
row_id = list(prediction_dict.keys())
 birds = list(prediction_dict.values())
 submission = pd.DataFrame({
"row_id": row_id,
"birds": birds
})
 submission.to_csv("submission.csv", index=False )<set_options>
new_col = []
for c in ind_agg.columns.levels[0]:
for stat in ind_agg.columns.levels[1]:
new_col.append(f'{c}-{stat}')
 ind_agg.columns = new_col
ind_agg.head()
pd.options.display.max_rows = 500
pd.options.display.max_columns = 500<load_pretrained>
ind_agg = ind_agg.drop(columns = to_drop)
 ind_feats = list(ind_agg.columns)
 final = heads.merge(ind_agg, on = 'idhogar', how = 'left')
 print('Final features shape: ', final.shape )
def load_audio(path, sr):
clip, _ = librosa.load(path, sr=sr, mono=True, res_type="kaiser_fast")
 return clip<load_pretrained>
scorer = make_scorer(f1_score, greater_is_better=True, average = 'macro' )
def load_model_weights(model, weights):
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 state_dict = torch.load(weights, map_location=device)
 model.load_state_dict(state_dict )<set_options>
train_labels = np.array(list(final[final['Target'].notnull() ]['Target'].astype(np.uint8)))
 train_set = final[final['Target'].notnull() ].drop(columns = ['Id', 'idhogar', 'Target'])
 test_set = final[final['Target'].isnull() ].drop(columns = ['Id', 'idhogar', 'Target'])
 submission_base = test[['Id', 'idhogar']].copy()
def seed_everything(seed):

 random.seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True<define_variables>
features = list(train_set.columns)
 pipeline = Pipeline([('imputer', Imputer(strategy = 'median')) , 
('scaler', MinMaxScaler())])
 train_set = pipeline.fit_transform(train_set)
 test_set = pipeline.transform(test_set )
SEED = 1213
seed_everything(SEED )<define_variables>
model = RandomForestClassifier(n_estimators=100, random_state=10,
n_jobs = -1)
 cv_score = cross_val_score(model, train_set, train_labels, cv = 10, scoring = scorer)
 print(f'10 Fold Cross Validation F1 Score = {round(cv_score.mean() , 4)} with std = {round(cv_score.std() , 4)}' )
ROOT = Path.cwd().parent
INPUT_ROOT = ROOT / "input"
RAW_DATA = INPUT_ROOT / "birdsong-recognition"
TRAIN_AUDIO_DIR = RAW_DATA / "train_audio"<load_from_csv>
model.fit(train_set, train_labels)
 feature_importances = pd.DataFrame({'feature': features, 'importance': model.feature_importances_})
 feature_importances.head()
train = pd.read_csv(RAW_DATA / "train.csv" )<load_from_csv>
from sklearn.svm import LinearSVC
from sklearn.naive_bayes import GaussianNB
from sklearn.neural_network import MLPClassifier
from sklearn.linear_model import LogisticRegressionCV, RidgeClassifierCV
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
from sklearn.neighbors import KNeighborsClassifier
TEST_AUDIO_DIR = RAW_DATA / "test_audio"
if not TEST_AUDIO_DIR.exists() :
TEST_AUDIO_DIR = INPUT_ROOT / "birdcall-check" / "test_audio"
test = pd.read_csv(INPUT_ROOT / "birdcall-check" / "test.csv")
 else:
test = pd.read_csv(RAW_DATA / "test.csv" )<define_variables>
model_results = cv_model(train_set, train_labels, LinearSVC() ,
'LSVC', model_results )
CLASSES = sorted(os.listdir(TRAIN_AUDIO_DIR))
NUM_CLASSES = len(CLASSES)
 NUM_WORKERS = 4<init_hyperparams>
model_results = cv_model(train_set, train_labels,
GaussianNB() , 'GNB', model_results )
class AudioParams:
sr = 32000
stride = 5
true_kernel_size = 5
img_size = None
n_mels = 128
fmin = 20
fmax = 16000<create_dataframe>
model_results = cv_model(train_set, train_labels,
MLPClassifier(hidden_layer_sizes=(32, 64, 128, 64, 32)) ,
'MLP', model_results )
def convert_site_3(df, clip_length, params):
n_samples = clip_length //(params.sr * params.true_kernel_size)
 audio_id = [df['audio_id'].values[0]] * n_samples
site = ['site_3'] * n_samples
seconds = [i * params.true_kernel_size for i in range(1, n_samples + 1)]
row_id = [f'site_3_{audio_id[0]}_{int(s)}' for s in seconds]
new_df = pd.DataFrame(data={'site': site,
'row_id': row_id,
'seconds': seconds,
'audio_id': audio_id
})
 return new_df<compute_train_metric>
model_results = cv_model(train_set, train_labels,
LinearDiscriminantAnalysis() ,
'LDA', model_results )
def compute_melspec(y, params):
melspec = librosa.feature.melspectrogram(
 y,
sr=params.sr,
n_mels=params.n_mels,
fmin=params.fmin,
fmax=params.fmax 
)
 melspec = librosa.power_to_db(melspec ).astype(np.float32)
 return melspec<categorify>
model_results = cv_model(train_set, train_labels,
RidgeClassifierCV() , 'RIDGE', model_results )
def mono_to_color(X, eps=1e-6, mean=None, std=None):
X = np.stack([X, X, X], axis=-1)
 mean = mean or X.mean()
std = std or X.std()
X =(X - mean)/(std + eps)
 _min, _max = X.min() , X.max()
if(_max - _min)> eps:
V = np.clip(X, _min, _max)
 V = 255 *(V - _min)/(_max - _min)
 V = V.astype(np.uint8)
 else:
V = np.zeros_like(X, dtype=np.uint8)
 return V
def resize(image, size=None):
if size is not None:
h, w, _ = image.shape
new_w, new_h = int(w * size / h), size
image = cv2.resize(image,(new_w, new_h))
return image
def normalize(image, mean=None, std=None):
image = image / 255.0
if mean is not None and std is not None:
image =(image - mean)/ std
return np.moveaxis(image, 2, 0 ).astype(np.float32 )<categorify>
for n in [5, 10, 20]:
print(f'
KNN with {n} neighbors
')
 model_results = cv_model(train_set, train_labels,
KNeighborsClassifier(n_neighbors = n),
f'knn-{n}', model_results )
class TestDataset(data.Dataset):
def __init__(self, df, clip, params):
self.df = df
self.clip = clip
self.params = params
if df['site'].values[0] == 'site_3':
self.df = convert_site_3(df, len(clip), params)
 def __len__(self):
return len(self.df)
 def __getitem__(self, idx):
end_seconds = int(self.df['seconds'][idx])
 start_seconds = int(end_seconds - 5)
 start_index = self.params.sr * start_seconds
end_index = self.params.sr * end_seconds
y = self.clip[start_index:end_index].astype(np.float32)
 melspec = compute_melspec(y, self.params)
 image = mono_to_color(melspec)
 image = resize(image, self.params.img_size)
 image = normalize(image, mean=None, std=None)
 return image<define_search_model>
model_results = cv_model(train_set, train_labels,
ExtraTreesClassifier(n_estimators = 100, random_state = 10),
'EXT', model_results )
class SplAtConv2d(Module):

 def __init__(self, in_channels, channels, kernel_size, stride=(1, 1), padding=(0, 0),
dilation=(1, 1), groups=1, bias=True,
radix=2, reduction_factor=4,
rectify=False, rectify_avg=False, norm_layer=None,
dropblock_prob=0.0, **kwargs):
super(SplAtConv2d, self ).__init__()
padding = _pair(padding)
 self.rectify = rectify and(padding[0] > 0 or padding[1] > 0)
 self.rectify_avg = rectify_avg
inter_channels = max(in_channels*radix//reduction_factor, 32)
 self.radix = radix
self.cardinality = groups
self.channels = channels
self.dropblock_prob = dropblock_prob
if self.rectify:
self.conv = RFConv2d(in_channels, channels*radix, kernel_size, stride, padding, dilation,
groups=groups*radix, bias=bias, average_mode=rectify_avg, **kwargs)
 else:
self.conv = Conv2d(in_channels, channels*radix, kernel_size, stride, padding, dilation,
groups=groups*radix, bias=bias, **kwargs)
 self.use_bn = norm_layer is not None
if self.use_bn:
self.bn0 = norm_layer(channels*radix)
 self.relu = ReLU(inplace=True)
 self.fc1 = Conv2d(channels, inter_channels, 1, groups=self.cardinality)
 if self.use_bn:
self.bn1 = norm_layer(inter_channels)
 self.fc2 = Conv2d(inter_channels, channels*radix, 1, groups=self.cardinality)
 if dropblock_prob > 0.0:
self.dropblock = DropBlock2D(dropblock_prob, 3)
 self.rsoftmax = rSoftMax(radix, groups)
 def forward(self, x):
x = self.conv(x)
 if self.use_bn:
x = self.bn0(x)
 if self.dropblock_prob > 0.0:
x = self.dropblock(x)
 x = self.relu(x)
 batch, rchannel = x.shape[:2]
if self.radix > 1:
if torch.__version__ < '1.5':
splited = torch.split(x, int(rchannel//self.radix), dim=1)
 else:
splited = torch.split(x, rchannel//self.radix, dim=1)
 gap = sum(splited)
 else:
gap = x
gap = F.adaptive_avg_pool2d(gap, 1)
 gap = self.fc1(gap)
 if self.use_bn:
gap = self.bn1(gap)
 gap = self.relu(gap)
 atten = self.fc2(gap)
 atten = self.rsoftmax(atten ).view(batch, -1, 1, 1)
 if self.radix > 1:
if torch.__version__ < '1.5':
attens = torch.split(atten, int(rchannel//self.radix), dim=1)
 else:
attens = torch.split(atten, rchannel//self.radix, dim=1)
 out = sum([att*split for(att, split)in zip(attens, splited)])
 else:
out = atten * x
return out.contiguous()
class rSoftMax(nn.Module):
def __init__(self, radix, cardinality):
super().__init__()
self.radix = radix
self.cardinality = cardinality
def forward(self, x):
batch = x.size(0)
 if self.radix > 1:
x = x.view(batch, self.cardinality, self.radix, -1 ).transpose(1, 2)
 x = F.softmax(x, dim=1)
 x = x.reshape(batch, -1)
 else:
x = torch.sigmoid(x)
 return x<import_modules>
model_results = cv_model(train_set, train_labels,
RandomForestClassifier(100, random_state=10),
'RF', model_results )
class DropBlock2D(object):
def __init__(self, *args, **kwargs):
raise NotImplementedError
class GlobalAvgPool2d(nn.Module):
def __init__(self):

 super(GlobalAvgPool2d, self ).__init__()
def forward(self, inputs):
return nn.functional.adaptive_avg_pool2d(inputs, 1 ).view(inputs.size(0), -1)
 class Bottleneck(nn.Module):

 expansion = 4
def __init__(self, inplanes, planes, stride=1, downsample=None,
radix=1, cardinality=1, bottleneck_width=64,
avd=False, avd_first=False, dilation=1, is_first=False,
rectified_conv=False, rectify_avg=False,
norm_layer=None, dropblock_prob=0.0, last_gamma=False):
super(Bottleneck, self ).__init__()
group_width = int(planes *(bottleneck_width / 64.))* cardinality
self.conv1 = nn.Conv2d(inplanes, group_width, kernel_size=1, bias=False)
 self.bn1 = norm_layer(group_width)
 self.dropblock_prob = dropblock_prob
self.radix = radix
self.avd = avd and(stride > 1 or is_first)
 self.avd_first = avd_first
if self.avd:
self.avd_layer = nn.AvgPool2d(3, stride, padding=1)
 stride = 1
if dropblock_prob > 0.0:
self.dropblock1 = DropBlock2D(dropblock_prob, 3)
 if radix == 1:
self.dropblock2 = DropBlock2D(dropblock_prob, 3)
 self.dropblock3 = DropBlock2D(dropblock_prob, 3)
 if radix >= 1:
self.conv2 = SplAtConv2d(
 group_width, group_width, kernel_size=3,
stride=stride, padding=dilation,
dilation=dilation, groups=cardinality, bias=False,
radix=radix, rectify=rectified_conv,
rectify_avg=rectify_avg,
norm_layer=norm_layer,
dropblock_prob=dropblock_prob)
 elif rectified_conv:
self.conv2 = RFConv2d(
 group_width, group_width, kernel_size=3, stride=stride,
padding=dilation, dilation=dilation,
groups=cardinality, bias=False,
average_mode=rectify_avg)
 self.bn2 = norm_layer(group_width)
 else:
self.conv2 = nn.Conv2d(
 group_width, group_width, kernel_size=3, stride=stride,
padding=dilation, dilation=dilation,
groups=cardinality, bias=False)
 self.bn2 = norm_layer(group_width)
 self.conv3 = nn.Conv2d(
 group_width, planes * 4, kernel_size=1, bias=False)
 self.bn3 = norm_layer(planes*4)
 if last_gamma:
zeros_(self.bn3.weight)
 self.relu = nn.ReLU(inplace=True)
 self.downsample = downsample
self.dilation = dilation
self.stride = stride
def forward(self, x):
residual = x
out = self.conv1(x)
 out = self.bn1(out)
 if self.dropblock_prob > 0.0:
out = self.dropblock1(out)
 out = self.relu(out)
 if self.avd and self.avd_first:
out = self.avd_layer(out)
 out = self.conv2(out)
 if self.radix == 0:
out = self.bn2(out)
 if self.dropblock_prob > 0.0:
out = self.dropblock2(out)
 out = self.relu(out)
 if self.avd and not self.avd_first:
out = self.avd_layer(out)
 out = self.conv3(out)
 out = self.bn3(out)
 if self.dropblock_prob > 0.0:
out = self.dropblock3(out)
 if self.downsample is not None:
residual = self.downsample(x)
 out += residual
out = self.relu(out)
 return out
class ResNet(nn.Module):

 def __init__(self, block, layers, radix=1, groups=1, bottleneck_width=64,
num_classes=1000, dilated=False, dilation=1,
deep_stem=False, stem_width=64, avg_down=False,
rectified_conv=False, rectify_avg=False,
avd=False, avd_first=False,
final_drop=0.0, dropblock_prob=0,
last_gamma=False, norm_layer=nn.BatchNorm2d):
self.cardinality = groups
self.bottleneck_width = bottleneck_width
self.inplanes = stem_width*2 if deep_stem else 64
self.avg_down = avg_down
self.last_gamma = last_gamma
self.radix = radix
self.avd = avd
self.avd_first = avd_first
super(ResNet, self ).__init__()
self.rectified_conv = rectified_conv
self.rectify_avg = rectify_avg
if rectified_conv:
conv_layer = RFConv2d
else:
conv_layer = nn.Conv2d
conv_kwargs = {'average_mode': rectify_avg} if rectified_conv else {}
if deep_stem:
self.conv1 = nn.Sequential(
 conv_layer(3, stem_width, kernel_size=3, stride=2, padding=1, bias=False, **conv_kwargs),
norm_layer(stem_width),
nn.ReLU(inplace=True),
conv_layer(stem_width, stem_width, kernel_size=3, stride=1, padding=1, bias=False, **conv_kwargs),
norm_layer(stem_width),
nn.ReLU(inplace=True),
conv_layer(stem_width, stem_width*2, kernel_size=3, stride=1, padding=1, bias=False, **conv_kwargs), 
)
 else:
self.conv1 = conv_layer(3, 64, kernel_size=7, stride=2, padding=3,
bias=False, **conv_kwargs)
 self.bn1 = norm_layer(self.inplanes)
 self.relu = nn.ReLU(inplace=True)
 self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
 self.layer1 = self._make_layer(block, 64, layers[0], norm_layer=norm_layer, is_first=False)
 self.layer2 = self._make_layer(block, 128, layers[1], stride=2, norm_layer=norm_layer)
 if dilated or dilation == 4:
self.layer3 = self._make_layer(block, 256, layers[2], stride=1,
dilation=2, norm_layer=norm_layer,
dropblock_prob=dropblock_prob)
 self.layer4 = self._make_layer(block, 512, layers[3], stride=1,
dilation=4, norm_layer=norm_layer,
dropblock_prob=dropblock_prob)
 elif dilation==2:
self.layer3 = self._make_layer(block, 256, layers[2], stride=2,
dilation=1, norm_layer=norm_layer,
dropblock_prob=dropblock_prob)
 self.layer4 = self._make_layer(block, 512, layers[3], stride=1,
dilation=2, norm_layer=norm_layer,
dropblock_prob=dropblock_prob)
 else:
self.layer3 = self._make_layer(block, 256, layers[2], stride=2,
norm_layer=norm_layer,
dropblock_prob=dropblock_prob)
 self.layer4 = self._make_layer(block, 512, layers[3], stride=2,
norm_layer=norm_layer,
dropblock_prob=dropblock_prob)
 self.avgpool = GlobalAvgPool2d()
self.drop = nn.Dropout(final_drop)if final_drop > 0.0 else None
self.fc = nn.Linear(512 * block.expansion, num_classes)
 for m in self.modules() :
if isinstance(m, nn.Conv2d):
n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
m.weight.data.normal_(0, math.sqrt(2./ n))
elif isinstance(m, norm_layer):
m.weight.data.fill_(1)
 m.bias.data.zero_()
def _make_layer(self, block, planes, blocks, stride=1, dilation=1, norm_layer=None,
dropblock_prob=0.0, is_first=True):
downsample = None
if stride != 1 or self.inplanes != planes * block.expansion:
down_layers = []
if self.avg_down:
if dilation == 1:
down_layers.append(nn.AvgPool2d(kernel_size=stride, stride=stride,
ceil_mode=True, count_include_pad=False))
else:
down_layers.append(nn.AvgPool2d(kernel_size=1, stride=1,
ceil_mode=True, count_include_pad=False))
down_layers.append(nn.Conv2d(self.inplanes, planes * block.expansion,
kernel_size=1, stride=1, bias=False))
else:
down_layers.append(nn.Conv2d(self.inplanes, planes * block.expansion,
kernel_size=1, stride=stride, bias=False))
down_layers.append(norm_layer(planes * block.expansion))
downsample = nn.Sequential(*down_layers)
 layers = []
if dilation == 1 or dilation == 2:
layers.append(block(self.inplanes, planes, stride, downsample=downsample,
radix=self.radix, cardinality=self.cardinality,
bottleneck_width=self.bottleneck_width,
avd=self.avd, avd_first=self.avd_first,
dilation=1, is_first=is_first, rectified_conv=self.rectified_conv,
rectify_avg=self.rectify_avg,
norm_layer=norm_layer, dropblock_prob=dropblock_prob,
last_gamma=self.last_gamma))
elif dilation == 4:
layers.append(block(self.inplanes, planes, stride, downsample=downsample,
radix=self.radix, cardinality=self.cardinality,
bottleneck_width=self.bottleneck_width,
avd=self.avd, avd_first=self.avd_first,
dilation=2, is_first=is_first, rectified_conv=self.rectified_conv,
rectify_avg=self.rectify_avg,
norm_layer=norm_layer, dropblock_prob=dropblock_prob,
last_gamma=self.last_gamma))
else:
raise RuntimeError("=> unknown dilation size: {}".format(dilation))
self.inplanes = planes * block.expansion
for i in range(1, blocks):
layers.append(block(self.inplanes, planes,
radix=self.radix, cardinality=self.cardinality,
bottleneck_width=self.bottleneck_width,
avd=self.avd, avd_first=self.avd_first,
dilation=dilation, rectified_conv=self.rectified_conv,
rectify_avg=self.rectify_avg,
norm_layer=norm_layer, dropblock_prob=dropblock_prob,
last_gamma=self.last_gamma))
return nn.Sequential(*layers)
 def forward(self, x):
x = self.conv1(x)
 x = self.bn1(x)
 x = self.relu(x)
 x = self.maxpool(x)
 x = self.layer1(x)
 x = self.layer2(x)
 x = self.layer3(x)
 x = self.layer4(x)
 x = self.avgpool(x)
 x = torch.flatten(x, 1)
 if self.drop:
x = self.drop(x)
 x = self.fc(x)
 return x<init_hyperparams>
def submit(model):
model.fit(train_set, train_labels)
 predictions = model.predict(test_set)
 predictions = pd.DataFrame({'idhogar': test_ids,
'Target': predictions})
 submission = submission_base.merge(predictions,
on = 'idhogar',
how = 'left' ).drop(columns = ['idhogar'])
 submission['Target'] = submission['Target'].fillna(4 ).astype(np.int8)
 return submission
MODEL_CONFIGS = {
"resnest50_fast_1s1x64d":
{
"num_classes": 264,
"block": Bottleneck,
"layers": [3, 4, 6, 3],
"radix": 1,
"groups": 1,
"bottleneck_width": 64,
"deep_stem": True,
"stem_width": 32,
"avg_down": True,
"avd" : True,
"avd_first" : True
},
"resnest101":
{
"num_classes": 264,
"block": Bottleneck,
"layers": [3, 4, 23, 3],
"radix": 2,
"groups": 1,
"bottleneck_width": 64,
"deep_stem": True,
"stem_width": 64,
"avg_down": True,
"avd" : True,
"avd_first" : False
},
}<choose_model_class>
X_train, X_test, y_train, y_test = train_test_split(train_set, train_labels, test_size = 0.2,
random_state =42 )
def get_model(name):
if "resnest" in name:
model = ResNet(**MODEL_CONFIGS[name])
 elif "resnext101" in name:
model = torchvision.models.resnext101_32x8d(pretrained=False)
 elif "resnext50" in name:
model = torchvision.models.resnext50_32x4d(pretrained=False)
 else:
raise NotImplementedError
nb_ft = model.fc.in_features
del model.fc
model.fc = nn.Linear(nb_ft, NUM_CLASSES)
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 model.to(device)
 model.eval()
return model<predict_on_test>
import xgboost
from xgboost import XGBClassifier
from sklearn.metrics import f1_score
def predict(model, dataset, batch_size=16):
model.eval()
preds = np.empty(( 0, NUM_CLASSES))
loader = DataLoader(
 dataset, batch_size=batch_size, shuffle=False, num_workers=NUM_WORKERS, pin_memory=True 
)
 with torch.no_grad() :
for x in loader:
y_pred = model(x.cuda() ).detach()
preds = np.concatenate([preds, torch.sigmoid(y_pred ).cpu().numpy() ])
 return preds<define_variables>
xgb_model = XGBClassifier(objective = 'multi:softmax', random_state = 42)
 xgb_model.fit(X_train, y_train)
 y_pred = xgb_model.predict(X_test)
 f1_score(y_test, y_pred, average = 'macro' )
def post_process_site_12(preds, threshold=0.5, maxpreds=3):
preds = preds *(preds >= threshold)
 next_preds = np.concatenate([preds[1:], np.zeros(( 1, preds.shape[-1])) ])
 prev_preds = np.concatenate([np.zeros(( 1, preds.shape[-1])) , preds[:-1]])
 score = preds + 0.5 * next_preds + 0.5 * prev_preds
n_birds =(score >= threshold ).sum(-1)
 n_birds = np.clip(n_birds, 0, maxpreds)
 labels = [np.argsort(- score[i])[:n_birds[i]] for i in range(len(preds)) ]
class_labels = [" ".join([CLASSES[l] for l in label])for label in labels]
return class_labels<groupby>
test_ids = list(final.loc[final['Target'].isnull() , 'idhogar'] )
def post_process_site_3(preds, threshold=0.5, maxpreds=3):
preds = preds *(preds >= threshold)
 score = np.sum(preds, 0)
 n_birds =(score >= threshold ).sum(-1)
 n_birds = np.clip(n_birds, 0, maxpreds)
 label = np.argsort(- score)[:n_birds]
class_labels = " ".join([CLASSES[l] for l in label])
 return class_labels<define_variables>
import lightgbm
from lightgbm import LGBMClassifier
def max_pred_gen(site, duration):
if site != "site_3":
return 3
else:
rets = [(7,2),(15, 3),(30, 5),(60, 7)]
for ref_duration,thresh in rets:
if ref_duration >= duration:
return thresh
return 10<create_dataframe>
lgb_model = LGBMClassifier(objective = 'multiclass', random_state = 42)
 lgb_model.fit(X_train, y_train)
 y_pred = lgb_model.predict(X_test)
 print(y_pred)
 f1_score(y_test, y_pred, average = 'macro' )
def reformat_preds(preds, df, site):
prediction_df = pd.DataFrame({
"row_id": df['row_id'].values,
"birds": preds
})
 prediction_df['birds'] = prediction_df['birds'].replace([''],'nocall')
 return prediction_df<load_pretrained>
xgb_submission.to_csv('xgb.csv', index = False)
 lgb_submission.to_csv('lgb.csv', index = False )
def inference(test_df, test_audio, configs, params, threshold=0.5):
unique_audio_id = test_df.audio_id.unique()
models = []
for config in configs:
models_ = []
for weights in config["weights"]:
model = get_model(config['name'])
 load_model_weights(model, weights)
 models_.append(model)
 models.append(models_)
 print(f'\t -> Using {len(models)} models, with {len(models[0])} weights per model.')
 pred_dfs = []
for audio_id in unique_audio_id :
audio_df = test_df[test_df['audio_id'] == audio_id].reset_index(drop=True)
 site = audio_df["site"].values[0]
print(f'
Making predictions for audio {audio_id} in {site} ')
 clip = load_audio(test_audio /(audio_id + ".mp3"), params.sr)
 clip_duration = len(clip)// params.sr
dataset = TestDataset(audio_df, clip, params)
 preds = []
for i, config in enumerate(configs):
for j, weights in enumerate(config["weights"]):
pred = predict(models[i][j], dataset, batch_size=16)
 preds.append(pred)
 preds = np.mean(preds, 0)
 maxpreds = max_pred_gen(site, clip_duration)
 print(f'Limiting the number of birds to {maxpreds}')
 if site == 'site_3':
preds_pp = post_process_site_3(preds, threshold=threshold, maxpreds=maxpreds)
 else:
preds_pp = post_process_site_12(preds, threshold=threshold, maxpreds=maxpreds)
 print("Predicted classes :", preds_pp)
 pred_df = reformat_preds(preds_pp, audio_df, site)
 pred_dfs.append(pred_df)
 sub = pd.concat(pred_dfs, axis=0, sort=False ).reset_index(drop=True)
 return sub<groupby>
print('Train start')
 params = {"objective" : "multiclass",
"num_class" : 4,
"num_leaves" : 60,
"max_depth": -1,
"learning_rate" : 0.01,
"bagging_fraction" : 0.9,
"feature_fraction" : 0.9,
"bagging_freq" : 5,
"bagging_seed" : 42,
"verbosity" : -1 }
def vote(preds, min_votes=3):
votes = Counter(preds)
 return [c for c, count in votes.items() if count >= min_votes]<categorify>
y_tra = y_train - 1
y_tes = y_test -1
def inference_voting(test_df, test_audio, configs, params, threshold=0.5, min_votes=3):
unique_audio_id = test_df.audio_id.unique()
models = []
for config in configs:
models_ = []
for weights in config["weights"]:
model = get_model(config['name'])
 load_model_weights(model, weights)
 models_.append(model)
 models.append(models_)
 print(f'\t -> Using {len(models)} models, with {len(models[0])} weights per model.')
 pred_dfs = []
for audio_id in unique_audio_id :
audio_df = test_df[test_df['audio_id'] == audio_id].reset_index(drop=True)
 site = audio_df["site"].values[0]
print(f'
Making predictions for audio {audio_id} in {site} ')
 clip = load_audio(test_audio /(audio_id + ".mp3"), params.sr)
 clip_duration = len(clip)// params.sr
dataset = TestDataset(audio_df, clip, params)
 all_preds = []
for i, config in enumerate(configs):
preds = []
for j, weights in enumerate(config["weights"]):
pred = predict(models[i][j], dataset, batch_size=16)
 preds.append(pred)
 preds = np.mean(preds, 0)
 maxpreds = max_pred_gen(site, clip_duration)
 print(f'Limiting the number of birds to {maxpreds}')
 if site == 'site_3':
preds_pp = post_process_site_3(preds, threshold=threshold, maxpreds=maxpreds)
 preds_pp = [preds_pp]
else:
preds_pp = post_process_site_12(preds, threshold=threshold, maxpreds=maxpreds)
 all_preds.append(preds_pp)
 print("Predicted classes :", preds_pp)
 final_preds = []
for i in range(len(all_preds[0])) :
preds = []
for m in range(len(all_preds)) :
preds += all_preds[m][i].split(' ')
 final_pred = vote(preds, min_votes=min_votes)
 final_preds.append(' '.join(final_pred))
print("
-> Voted classes :", final_preds)
 pred_df = reformat_preds(final_preds, audio_df, site)
 pred_dfs.append(pred_df)
 sub = pd.concat(pred_dfs, axis=0, sort=False ).reset_index(drop=True)
 return sub<define_variables>
lgtrain, lgval = lgb.Dataset(data = X_train, label = y_tra), lgb.Dataset(data = X_test, label = y_tes)
 print(lgtrain)
 lgb_ad_model = lgb.train(params, lgtrain, num_boost_round = 2000, valid_sets=[lgtrain, lgval],
early_stopping_rounds=100, verbose_eval=100 )
configs = []<load_pretrained>
predictions = lgb_ad_model.predict(test_set )
model_name = "resnext50_32x4d"
weights = [f".. /input/birds-cp-2/{model_name}_extra_{i}.pt" for i in range(5)]
for w in weights:
assert os.path.isfile(w), f"Weights {w} not found"
configs.append({
"name": model_name,
"weights": weights,
} )<define_variables>
predict_df_1 = []
predict_df_2 = []
predict_df_3 = []
predict_df_4 = []
model_name = "resnext101_32x8d_wsl"
weights = [f".. /input/birds-cp-2/{model_name}_extra_{i}.pt" for i in range(5)]
for w in weights:
assert os.path.isfile(w), f"Weights {w} not found"
configs.append({
"name": model_name,
"weights": weights,
} )<define_variables>
for i in range(len(predictions)) :
predict_df_1.append(predictions[i][0])
 predict_df_2.append(predictions[i][1])
 predict_df_3.append(predictions[i][2])
 predict_df_4.append(predictions[i][3] )
model_name = "resnest50_fast_1s1x64d"
weights = [f".. /input/birds-cp-1/{model_name}_conf_{i}.pt" for i in range(5)]
for w in weights:
assert os.path.isfile(w), f"Weights {w} not found"
configs.append({
"name": model_name,
"weights": weights,
} )<define_variables>
df_pred = pd.DataFrame(predict_df_1, columns = ['1'] )
threshold = 0.5
min_votes = 2<split>
df_pred['2'] = predict_df_2
df_pred['3'] = predict_df_3
df_pred['4'] = predict_df_4
df_pred.head()
warnings.filterwarnings("ignore")
 submission = inference_voting(test, TEST_AUDIO_DIR, configs, AudioParams, threshold=threshold, min_votes=min_votes )<save_to_csv>
df_pred['Target'] = df_pred[['1', '2', '3', '4']].idxmax(axis = 1 )
submission.to_csv("submission.csv", index=False)
<import_modules>
df_pred.to_csv('result.csv', index = False )