Kaggle competition: Cornell Birdcall Identification
<SOS> metric: MeanFScoreBeta Kaggle data source: cornell-birdcall-identification<rename_columns>
!pip install.. /input/bird-panns/torchlibrosa-master/torchlibrosa-master/
x_develop.set_index('sig_id', inplace=True)
 x_test.set_index('sig_id', inplace=True )<predict_on_test>
pd.options.display.max_rows = 500
pd.options.display.max_columns = 500
def calc_pred(x_df, models):
if isinstance(models[0], np.ndarray):
pred_test = np.repeat([models[0]], len(x_df), axis=0)
 else:
for i in models:
if i == 0:
pred_test = models[i].predict(x_df)
 else:
pred_test += models[i].predict(x_df)
 pred_test = pred_test/len(models)
 pred_test = pd.DataFrame(index=x_df.index, columns=target_cols, data=pred_test)
 pred_test.loc[x_df['cp_type'] == 0] = 0
return pred_test
def oof_score(oof: dict):
return np.mean(list(oof.values())) , np.std(list(oof.values()))
def combine_pred(pred):
for i in pred:
if i==0:
r = pred[i]
else:
r = np.append(r, pred[i], axis=0)
 return r
def run_cv(xtrain=x_develop, ytrain=y_develop, model=None, N_FOLDS=N_FOLDS, summary=True, debug=False):
histories = {x: '' for x in range(N_FOLDS)}
models = {x: '' for x in range(N_FOLDS)}
results = {x: '' for x in range(N_FOLDS)}
oof_bp = {x: [] for x in range(N_FOLDS)}
oof_ap = {x: [] for x in range(N_FOLDS)}
pred_val_fold = {x: [] for x in range(N_FOLDS)}
target_val_fold = {x: [] for x in range(N_FOLDS)}
oof_pd = pd.DataFrame()
for foldno in np.sort(y_develop['Fold'].unique()):
x_train_fold = x_develop[y_develop['Fold'] != foldno]
y_train_fold = y_develop[y_develop['Fold'] != foldno].drop('Fold', axis=1)
 x_val_fold = x_develop[y_develop['Fold'] == foldno]
y_val_fold = y_develop[y_develop['Fold'] == foldno].drop('Fold', axis=1)
 train_sample_size = len(y_train_fold)
 val_sample_size = len(y_val_fold)
 print(" ")
 print(f"Fold-%d" %(foldno))
print("Train sample size:", train_sample_size, ", Validation sample size:", val_sample_size)
 FEATURE_SIZE = x_train_fold.shape[-1]
train_ds = tf.data.Dataset.from_tensor_slices(( x_train_fold, y_train_fold))
train_ds = train_ds.shuffle(1024 ).batch(56)
 val_ds = tf.data.Dataset.from_tensor_slices(( x_val_fold, y_val_fold))
val_ds = val_ds.batch(val_sample_size)
 models[foldno] = load_model(f'.. /input/update-mx10-model/weights-fold{foldno}.h5', compile=False)
 pred_val_fold[foldno] = models[foldno].predict(x_val_fold)
 oof_bp[foldno] = tf.keras.losses.binary_crossentropy(y_val_fold, pred_val_fold[foldno] ).numpy().mean()
print('Out-of-Fold Score: ', oof_bp[foldno])
 pred_val_fold[foldno][x_val_fold['cp_type'] == 0] == 0
oof_ap[foldno] = tf.keras.losses.binary_crossentropy(y_val_fold, pred_val_fold[foldno] ).numpy().mean()
print('Out-of-Fold Score with post processing: ', oof_ap[foldno])
 target_val_fold[foldno] = y_val_fold.to_numpy()
cur_oof_pd = pd.DataFrame(pred_val_fold[foldno], columns=[f'{i}' for i in range(206)])
 cur_oof_pd['sig_id'] = y_val_fold.index
oof_pd = oof_pd.append(cur_oof_pd)
 if SAVE_MODEL:
pass
pred_val_fold = combine_pred(pred_val_fold)
 target_val_fold = combine_pred(target_val_fold)
 print('
')
 if summary:
print('Summary')
 print('Mean OOF score: %f +/- %f' %(oof_score(oof_bp)))
 print('Mean OOF score after postprocessing: %f +/- %f' %(oof_score(oof_ap)))
 return models, histories, oof_ap, pred_val_fold, pred_val_fold, target_val_fold, oof_pd
def submit(res):
sub = res.reset_index()
sub.to_csv('submission.csv', index=False )<define_variables>
def set_seed(seed: int = 42):
random.seed(seed)
 np.random.seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 @contextmanager
def timer(name: str)-> None:

 t0 = time.time()
print("[{}] start".format(name))
yield
print("[{}] done in {:.0f} s".format(name, time.time() - t0))
EPOCHS = 45
SAVE_MODEL = True<compute_train_metric>
set_seed(42 )
output_bias = -np.log(y_develop[y_develop.columns[:-1]].mean(axis=0 ).to_numpy())
 models, histories, oof_ap, pred_val_fold, pred_val_fold, target_val_fold, oof_pd = run_cv(x_develop, y_develop,
model=Model(x_develop.shape[1], output_bias),
N_FOLDS=N_FOLDS,
debug=False)
 pred_test = calc_pred(x_test, models)
 submit(pred_test )<compute_test_metric>
SR = 32000
ROOT = Path.cwd().parent
INPUT_ROOT = ROOT / "input"
RAW_DATA = INPUT_ROOT / "birdsong-recognition"
TRAIN_AUDIO_DIR = RAW_DATA / "train_audio"
TEST_AUDIO_DIR = RAW_DATA / "test_audio"
def log_loss_score(actual, predicted, eps=1e-15):
p1 = actual * np.log(predicted+eps)
 p0 =(1-actual)* np.log(1-predicted+eps)
 loss = p0 + p1
return -loss.mean()
def log_loss_multi(y_true, y_pred):
M = y_true.shape[1]
results = np.zeros(M)
 for i in range(M):
results[i] = log_loss_score(y_true[:,i], y_pred[:,i])
 return results.mean()
oof_final = y_develop.merge(oof_pd, how='left', on='sig_id')[[str(col)for col in range(206)]]<load_from_csv>
train = pd.read_csv(RAW_DATA / "train.csv" )
y_true = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 log_loss_multi(y_true.drop(['sig_id'], axis=1 ).to_numpy() , oof_final.to_numpy() )<data_type_conversions>
if not TEST_AUDIO_DIR.exists() :
TEST_AUDIO_DIR = INPUT_ROOT / "birdcall-check" / "test_audio"
test = pd.read_csv(INPUT_ROOT / "birdcall-check" / "test.csv")
 else:
test = pd.read_csv(RAW_DATA / "test.csv" )
np.save('LBS.npy', oof_final.to_numpy() )<set_options>
sub = pd.read_csv(".. /input/birdsong-recognition/sample_submission.csv")
 sub.to_csv("submission.csv", index=False )
warnings.filterwarnings('ignore')
 BOOK1 = '.. /input/daebook1c/'
BOOK2 = '.. /input/daebook2c/'
CTT = 7<set_options>
BIRD_CODE = {
'aldfly': 0, 'ameavo': 1, 'amebit': 2, 'amecro': 3, 'amegfi': 4,
'amekes': 5, 'amepip': 6, 'amered': 7, 'amerob': 8, 'amewig': 9,
'amewoo': 10, 'amtspa': 11, 'annhum': 12, 'astfly': 13, 'baisan': 14,
'baleag': 15, 'balori': 16, 'banswa': 17, 'barswa': 18, 'bawwar': 19,
'belkin1': 20, 'belspa2': 21, 'bewwre': 22, 'bkbcuc': 23, 'bkbmag1': 24,
'bkbwar': 25, 'bkcchi': 26, 'bkchum': 27, 'bkhgro': 28, 'bkpwar': 29,
'bktspa': 30, 'blkpho': 31, 'blugrb1': 32, 'blujay': 33, 'bnhcow': 34,
'boboli': 35, 'bongul': 36, 'brdowl': 37, 'brebla': 38, 'brespa': 39,
'brncre': 40, 'brnthr': 41, 'brthum': 42, 'brwhaw': 43, 'btbwar': 44,
'btnwar': 45, 'btywar': 46, 'buffle': 47, 'buggna': 48, 'buhvir': 49,
'bulori': 50, 'bushti': 51, 'buwtea': 52, 'buwwar': 53, 'cacwre': 54,
'calgul': 55, 'calqua': 56, 'camwar': 57, 'cangoo': 58, 'canwar': 59,
'canwre': 60, 'carwre': 61, 'casfin': 62, 'caster1': 63, 'casvir': 64,
'cedwax': 65, 'chispa': 66, 'chiswi': 67, 'chswar': 68, 'chukar': 69,
'clanut': 70, 'cliswa': 71, 'comgol': 72, 'comgra': 73, 'comloo': 74,
'commer': 75, 'comnig': 76, 'comrav': 77, 'comred': 78, 'comter': 79,
'comyel': 80, 'coohaw': 81, 'coshum': 82, 'cowscj1': 83, 'daejun': 84,
'doccor': 85, 'dowwoo': 86, 'dusfly': 87, 'eargre': 88, 'easblu': 89,
'easkin': 90, 'easmea': 91, 'easpho': 92, 'eastow': 93, 'eawpew': 94,
'eucdov': 95, 'eursta': 96, 'evegro': 97, 'fiespa': 98, 'fiscro': 99,
'foxspa': 100, 'gadwal': 101, 'gcrfin': 102, 'gnttow': 103, 'gnwtea': 104,
'gockin': 105, 'gocspa': 106, 'goleag': 107, 'grbher3': 108, 'grcfly': 109,
'greegr': 110, 'greroa': 111, 'greyel': 112, 'grhowl': 113, 'grnher': 114,
'grtgra': 115, 'grycat': 116, 'gryfly': 117, 'haiwoo': 118, 'hamfly': 119,
'hergul': 120, 'herthr': 121, 'hoomer': 122, 'hoowar': 123, 'horgre': 124,
'horlar': 125, 'houfin': 126, 'houspa': 127, 'houwre': 128, 'indbun': 129,
'juntit1': 130, 'killde': 131, 'labwoo': 132, 'larspa': 133, 'lazbun': 134,
'leabit': 135, 'leafly': 136, 'leasan': 137, 'lecthr': 138, 'lesgol': 139,
'lesnig': 140, 'lesyel': 141, 'lewwoo': 142, 'linspa': 143, 'lobcur': 144,
'lobdow': 145, 'logshr': 146, 'lotduc': 147, 'louwat': 148, 'macwar': 149,
'magwar': 150, 'mallar3': 151, 'marwre': 152, 'merlin': 153, 'moublu': 154,
'mouchi': 155, 'moudov': 156, 'norcar': 157, 'norfli': 158, 'norhar2': 159,
'normoc': 160, 'norpar': 161, 'norpin': 162, 'norsho': 163, 'norwat': 164,
'nrwswa': 165, 'nutwoo': 166, 'olsfly': 167, 'orcwar': 168, 'osprey': 169,
'ovenbi1': 170, 'palwar': 171, 'pasfly': 172, 'pecsan': 173, 'perfal': 174,
'phaino': 175, 'pibgre': 176, 'pilwoo': 177, 'pingro': 178, 'pinjay': 179,
'pinsis': 180, 'pinwar': 181, 'plsvir': 182, 'prawar': 183, 'purfin': 184,
'pygnut': 185, 'rebmer': 186, 'rebnut': 187, 'rebsap': 188, 'rebwoo': 189,
'redcro': 190, 'redhea': 191, 'reevir1': 192, 'renpha': 193, 'reshaw': 194,
'rethaw': 195, 'rewbla': 196, 'ribgul': 197, 'rinduc': 198, 'robgro': 199,
'rocpig': 200, 'rocwre': 201, 'rthhum': 202, 'ruckin': 203, 'rudduc': 204,
'rufgro': 205, 'rufhum': 206, 'rusbla': 207, 'sagspa1': 208, 'sagthr': 209,
'savspa': 210, 'saypho': 211, 'scatan': 212, 'scoori': 213, 'semplo': 214,
'semsan': 215, 'sheowl': 216, 'shshaw': 217, 'snobun': 218, 'snogoo': 219,
'solsan': 220, 'sonspa': 221, 'sora': 222, 'sposan': 223, 'spotow': 224,
'stejay': 225, 'swahaw': 226, 'swaspa': 227, 'swathr': 228, 'treswa': 229,
'truswa': 230, 'tuftit': 231, 'tunswa': 232, 'veery': 233, 'vesspa': 234,
'vigswa': 235, 'warvir': 236, 'wesblu': 237, 'wesgre': 238, 'weskin': 239,
'wesmea': 240, 'wessan': 241, 'westan': 242, 'wewpew': 243, 'whbnut': 244,
'whcspa': 245, 'whfibi': 246, 'whtspa': 247, 'whtswi': 248, 'wilfly': 249,
'wilsni1': 250, 'wiltur': 251, 'winwre3': 252, 'wlswar': 253, 'wooduc': 254,
'wooscj2': 255, 'woothr': 256, 'y00475': 257, 'yebfly': 258, 'yebsap': 259,
'yehbla': 260, 'yelwar': 261, 'yerwar': 262, 'yetvir': 263
}
INV_BIRD_CODE = {v: k for k, v in BIRD_CODE.items() }
def reduce_mem_usage(props):
start_mem_usg = props.memory_usage().sum() / 1024**2
print("Memory usage of properties dataframe is :",start_mem_usg," MB")
 NAlist = []
for col in props.columns:
if props[col].dtype != object:
IsInt = False
mx = props[col].max()
mn = props[col].min()
if not np.isfinite(props[col] ).all() :
NAlist.append(col)
 props[col].fillna(mn-1,inplace=True)
 asint = props[col].fillna(0 ).astype(np.int64)
 result =(props[col] - asint)
 result = result.sum()
if result > -0.01 and result < 0.01:
IsInt = True
if IsInt:
if mn >= 0:
if mx < 255:
props[col] = props[col].astype(np.uint8)
 elif mx < 65535:
props[col] = props[col].astype(np.uint16)
 elif mx < 4294967295:
props[col] = props[col].astype(np.uint32)
 else:
props[col] = props[col].astype(np.uint64)
 else:
if mn > np.iinfo(np.int8 ).min and mx < np.iinfo(np.int8 ).max:
props[col] = props[col].astype(np.int8)
 elif mn > np.iinfo(np.int16 ).min and mx < np.iinfo(np.int16 ).max:
props[col] = props[col].astype(np.int16)
 elif mn > np.iinfo(np.int32 ).min and mx < np.iinfo(np.int32 ).max:
props[col] = props[col].astype(np.int32)
 elif mn > np.iinfo(np.int64 ).min and mx < np.iinfo(np.int64 ).max:
props[col] = props[col].astype(np.int64)
 else:
props[col] = props[col].astype(np.float32)
 print("___MEMORY USAGE AFTER COMPLETION:___")
 mem_usg = props.memory_usage().sum() / 1024**2
print("Memory usage is: ",mem_usg," MB")
 print("This is ",100*mem_usg/start_mem_usg,"% of the initial size")
 return props, NAlist
def train_short_form_loader(feature_file,target_file,extra_target_file=None):

 train_features = pd.read_csv(feature_file)
 train_targets = pd.read_csv(target_file)
 train_features,_= reduce_mem_usage(train_features)
 train_targets,_ = reduce_mem_usage(train_targets)
 if extra_target_file is not None:
extra_targets = pd.read_csv(extra_target_file)
 extra_targets,_ = reduce_mem_usage(extra_targets)
 train_targets = pd.merge(train_targets,extra_targets,on ='sig_id')
 del extra_targets
targets = train_targets.columns[1:]
train_melt=train_targets.merge(train_features,how="left",on="sig_id")
 del train_features,train_targets
train_melt.set_index("sig_id",inplace=True)
 train_melt["cp_type"]= train_melt["cp_type"].astype('category')
 train_melt["cp_dose"]= train_melt["cp_dose"].astype('category')
 return train_melt , targets.to_list()
def test_short_form_loader(feature_file):

 train_features = pd.read_csv(feature_file)
 train_features,_= reduce_mem_usage(train_features)
 train_melt = train_features.copy()
del train_features
train_melt.set_index("sig_id",inplace=True)
 train_melt["cp_type"]= train_melt["cp_type"].astype('category')
 train_melt["cp_dose"]= train_melt["cp_dose"].astype('category')
 return train_melt 
<define_variables>
class TestDataset(data.Dataset):
def __init__(self, df: pd.DataFrame, clip: np.ndarray):
self.df = df
self.clip = clip
def __len__(self):
return len(self.df)
 def __getitem__(self, idx: int):
SR = 32000
sample = self.df.loc[idx, :]
site = sample.site
row_id = sample.row_id
if site == "site_3":
y = self.clip.astype(np.float32)
 len_y = len(y)
 start = 0
end = SR * 5
y_all = []
while len_y > start:
y_batch = y[start:end].astype(np.float32)
 if len(y_batch)!=(SR * 5):
y_pad = np.zeros(5 * SR, dtype=np.float32)
 y_pad[:len(y_batch)] = y_batch
y_all.append(y_pad)
 break
start = end
end = end + SR * 5
y_all.append(y_batch)
 y_all = np.asarray(y_all)
 return y_all, row_id, site
else:
end_seconds = int(sample.seconds)
 start_seconds = int(end_seconds - 5)
 start_index = SR * start_seconds
end_index = SR * end_seconds
y = self.clip[start_index:end_index].astype(np.float32)
 return y, row_id, site
input_directory = '.. /input/lish-moa/'<load_from_csv>
def init_layer(layer):

 nn.init.xavier_uniform_(layer.weight)
 if hasattr(layer, 'bias'):
if layer.bias is not None:
layer.bias.data.fill_(0.)
def init_bn(bn):

 bn.bias.data.fill_(0.)
bn.weight.data.fill_(1.)
def _resnet_conv3x3(in_planes, out_planes):
return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=1,
padding=1, groups=1, bias=False, dilation=1)
 def _resnet_conv1x1(in_planes, out_planes):
return nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=1, bias=False)
 class ConvBlock(nn.Module):
def __init__(self, in_channels, out_channels):
super(ConvBlock, self ).__init__()
self.conv1 = nn.Conv2d(in_channels=in_channels,
out_channels=out_channels,
kernel_size=(3, 3), stride=(1, 1),
padding=(1, 1), bias=False)
 self.conv2 = nn.Conv2d(in_channels=out_channels,
out_channels=out_channels,
kernel_size=(3, 3), stride=(1, 1),
padding=(1, 1), bias=False)
 self.bn1 = nn.BatchNorm2d(out_channels)
 self.bn2 = nn.BatchNorm2d(out_channels)
 self.init_weight()
def init_weight(self):
init_layer(self.conv1)
 init_layer(self.conv2)
 init_bn(self.bn1)
 init_bn(self.bn2)
 def forward(self, input, pool_size=(2, 2), pool_type='avg'):
x = input
x = F.relu_(self.bn1(self.conv1(x)))
 x = F.relu_(self.bn2(self.conv2(x)))
 if pool_type == 'max':
x = F.max_pool2d(x, kernel_size=pool_size)
 elif pool_type == 'avg':
x = F.avg_pool2d(x, kernel_size=pool_size)
 elif pool_type == 'avg+max':
x1 = F.avg_pool2d(x, kernel_size=pool_size)
 x2 = F.max_pool2d(x, kernel_size=pool_size)
 x = x1 + x2
else:
raise Exception('Incorrect argument!')
 return x
class _ResnetBasicBlock(nn.Module):
expansion = 1
def __init__(self, inplanes, planes, stride=1, downsample=None, groups=1,
base_width=64, dilation=1, norm_layer=None):
super(_ResnetBasicBlock, self ).__init__()
if norm_layer is None:
norm_layer = nn.BatchNorm2d
if groups != 1 or base_width != 64:
raise ValueError('_ResnetBasicBlock only supports groups=1 and base_width=64')
 if dilation > 1:
raise NotImplementedError("Dilation > 1 not supported in _ResnetBasicBlock")
 self.stride = stride
self.conv1 = _resnet_conv3x3(inplanes, planes)
 self.bn1 = norm_layer(planes)
 self.relu = nn.ReLU(inplace=True)
 self.conv2 = _resnet_conv3x3(planes, planes)
 self.bn2 = norm_layer(planes)
 self.downsample = downsample
self.stride = stride
self.init_weights()
def init_weights(self):
init_layer(self.conv1)
 init_bn(self.bn1)
 init_layer(self.conv2)
 init_bn(self.bn2)
 nn.init.constant_(self.bn2.weight, 0)
 def forward(self, x):
identity = x
if self.stride == 2:
out = F.avg_pool2d(x, kernel_size=(2, 2))
else:
out = x
out = self.conv1(out)
 out = self.bn1(out)
 out = self.relu(out)
 out = F.dropout(out, p=0.1, training=self.training)
 out = self.conv2(out)
 out = self.bn2(out)
 if self.downsample is not None:
identity = self.downsample(identity)
 out += identity
out = self.relu(out)
 return out
class _ResNet(nn.Module):
def __init__(self, block, layers, zero_init_residual=False,
groups=1, width_per_group=64, replace_stride_with_dilation=None,
norm_layer=None):
super(_ResNet, self ).__init__()
if norm_layer is None:
norm_layer = nn.BatchNorm2d
self._norm_layer = norm_layer
self.inplanes = 64
self.dilation = 1
if replace_stride_with_dilation is None:
replace_stride_with_dilation = [False, False, False]
if len(replace_stride_with_dilation)!= 3:
raise ValueError("replace_stride_with_dilation should be None "
"or a 3-element tuple, got {}".format(replace_stride_with_dilation))
self.groups = groups
self.base_width = width_per_group
self.layer1 = self._make_layer(block, 64, layers[0], stride=1)
 self.layer2 = self._make_layer(block, 128, layers[1], stride=2,
dilate=replace_stride_with_dilation[0])
 self.layer3 = self._make_layer(block, 256, layers[2], stride=2,
dilate=replace_stride_with_dilation[1])
 self.layer4 = self._make_layer(block, 512, layers[3], stride=2,
dilate=replace_stride_with_dilation[2])
 def _make_layer(self, block, planes, blocks, stride=1, dilate=False):
norm_layer = self._norm_layer
downsample = None
previous_dilation = self.dilation
if dilate:
self.dilation *= stride
stride = 1
if stride != 1 or self.inplanes != planes * block.expansion:
if stride == 1:
downsample = nn.Sequential(
 _resnet_conv1x1(self.inplanes, planes * block.expansion),
norm_layer(planes * block.expansion), 
)
 init_layer(downsample[0])
 init_bn(downsample[1])
 elif stride == 2:
downsample = nn.Sequential(
 nn.AvgPool2d(kernel_size=2),
_resnet_conv1x1(self.inplanes, planes * block.expansion),
norm_layer(planes * block.expansion), 
)
 init_layer(downsample[1])
 init_bn(downsample[2])
 layers = []
layers.append(block(self.inplanes, planes, stride, downsample, self.groups,
self.base_width, previous_dilation, norm_layer))
self.inplanes = planes * block.expansion
for _ in range(1, blocks):
layers.append(block(self.inplanes, planes, groups=self.groups,
base_width=self.base_width, dilation=self.dilation,
norm_layer=norm_layer))
return nn.Sequential(*layers)
 def forward(self, x):
x = self.layer1(x)
 x = self.layer2(x)
 x = self.layer3(x)
 x = self.layer4(x)
 return x
class ResNet38(nn.Module):
def __init__(self, sample_rate, window_size, hop_size, mel_bins, fmin, fmax, classes_num):
super(ResNet38, self ).__init__()
window = 'hann'
center = True
pad_mode = 'reflect'
ref = 1.0
amin = 1e-10
top_db = None
self.spectrogram_extractor = Spectrogram(n_fft=window_size, hop_length=hop_size,
win_length=window_size, window=window, center=center, pad_mode=pad_mode,
freeze_parameters=True)
 self.logmel_extractor = LogmelFilterBank(sr=sample_rate, n_fft=window_size,
n_mels=mel_bins, fmin=fmin, fmax=fmax, ref=ref, amin=amin, top_db=top_db,
freeze_parameters=True)
 self.spec_augmenter = SpecAugmentation(time_drop_width=64, time_stripes_num=2,
freq_drop_width=8, freq_stripes_num=2)
 self.bn0 = nn.BatchNorm2d(64)
 self.conv_block1 = ConvBlock(in_channels=1, out_channels=64)
 self.resnet = _ResNet(block=_ResnetBasicBlock, layers=[3, 4, 6, 3], zero_init_residual=True)
 self.conv_block_after1 = ConvBlock(in_channels=512, out_channels=2048)
 self.fc1 = nn.Linear(2048, 2048)
 self.fc_audioset = nn.Linear(2048, classes_num, bias=True)
 self.init_weights()
def init_weights(self):
init_bn(self.bn0)
 init_layer(self.fc1)
 init_layer(self.fc_audioset)
 def forward(self, input, mixup_lambda=None):

 x = self.spectrogram_extractor(input)
 x = self.logmel_extractor(x)
 x = x.transpose(1, 3)
 x = self.bn0(x)
 x = x.transpose(1, 3)
 if self.training:
x = self.spec_augmenter(x)
 if self.training and mixup_lambda is not None:
x = do_mixup(x, mixup_lambda)
 x = self.conv_block1(x, pool_size=(2, 2), pool_type='avg')
 x = F.dropout(x, p=0.2, training=self.training, inplace=True)
 x = self.resnet(x)
 x = F.avg_pool2d(x, kernel_size=(2, 2))
x = F.dropout(x, p=0.2, training=self.training, inplace=True)
 x = self.conv_block_after1(x, pool_size=(1, 1), pool_type='avg')
 x = F.dropout(x, p=0.2, training=self.training, inplace=True)
 x = torch.mean(x, dim=3)
(x1, _)= torch.max(x, dim=2)
 x2 = torch.mean(x, dim=2)
 x = x1 + x2
x = F.dropout(x, p=0.5, training=self.training)
 x = F.relu_(self.fc1(x))
embedding = F.dropout(x, p=0.5, training=self.training)
 clipwise_output = torch.sigmoid(self.fc_audioset(x))
output_dict = {'clipwise_output': clipwise_output, 'embedding': embedding}
return output_dict
train,target_cols = train_short_form_loader(input_directory +'train_features.csv',input_directory+'train_targets_scored.csv')
 test = test_short_form_loader(input_directory +"test_features.csv")
<define_variables>
class ConvPreWavBlock(nn.Module):
def __init__(self, in_channels, out_channels):
super(ConvPreWavBlock, self ).__init__()
self.conv1 = nn.Conv1d(in_channels=in_channels,
out_channels=out_channels,
kernel_size=3, stride=1,
padding=1, bias=False)
 self.conv2 = nn.Conv1d(in_channels=out_channels,
out_channels=out_channels,
kernel_size=3, stride=1, dilation=2,
padding=2, bias=False)
 self.bn1 = nn.BatchNorm1d(out_channels)
 self.bn2 = nn.BatchNorm1d(out_channels)
 self.init_weight()
def init_weight(self):
init_layer(self.conv1)
 init_layer(self.conv2)
 init_bn(self.bn1)
 init_bn(self.bn2)
 def forward(self, input, pool_size):
x = input
x = F.relu_(self.bn1(self.conv1(x)))
 x = F.relu_(self.bn2(self.conv2(x)))
 x = F.max_pool1d(x, kernel_size=pool_size)
 return x
class Wavegram_Logmel_Cnn14(nn.Module):
def __init__(self, sample_rate, window_size, hop_size, mel_bins, fmin,
fmax, classes_num):
super(Wavegram_Logmel_Cnn14, self ).__init__()
window = 'hann'
center = True
pad_mode = 'reflect'
ref = 1.0
amin = 1e-10
top_db = None
self.pre_conv0 = nn.Conv1d(in_channels=1, out_channels=64, kernel_size=11, stride=5, padding=5, bias=False)
 self.pre_bn0 = nn.BatchNorm1d(64)
 self.pre_block1 = ConvPreWavBlock(64, 64)
 self.pre_block2 = ConvPreWavBlock(64, 128)
 self.pre_block3 = ConvPreWavBlock(128, 128)
 self.pre_block4 = ConvBlock(in_channels=4, out_channels=64)
 self.spectrogram_extractor = Spectrogram(n_fft=window_size, hop_length=hop_size,
win_length=window_size, window=window, center=center, pad_mode=pad_mode,
freeze_parameters=True)
 self.logmel_extractor = LogmelFilterBank(sr=sample_rate, n_fft=window_size,
n_mels=mel_bins, fmin=fmin, fmax=fmax, ref=ref, amin=amin, top_db=top_db,
freeze_parameters=True)
 self.spec_augmenter = SpecAugmentation(time_drop_width=64, time_stripes_num=2,
freq_drop_width=8, freq_stripes_num=2)
 self.bn0 = nn.BatchNorm2d(64)
 self.conv_block1 = ConvBlock(in_channels=1, out_channels=64)
 self.conv_block2 = ConvBlock(in_channels=128, out_channels=128)
 self.conv_block3 = ConvBlock(in_channels=128, out_channels=256)
 self.conv_block4 = ConvBlock(in_channels=256, out_channels=512)
 self.conv_block5 = ConvBlock(in_channels=512, out_channels=1024)
 self.conv_block6 = ConvBlock(in_channels=1024, out_channels=2048)
 self.fc1 = nn.Linear(2048, 2048, bias=True)
 self.fc_audioset = nn.Linear(2048, classes_num, bias=True)
 self.init_weight()
def init_weight(self):
init_layer(self.pre_conv0)
 init_bn(self.pre_bn0)
 init_bn(self.bn0)
 init_layer(self.fc1)
 init_layer(self.fc_audioset)
 def forward(self, input, mixup_lambda=None):

 a1 = F.relu_(self.pre_bn0(self.pre_conv0(input[:, None, :])))
 a1 = self.pre_block1(a1, pool_size=4)
 a1 = self.pre_block2(a1, pool_size=4)
 a1 = self.pre_block3(a1, pool_size=4)
 a1 = a1.reshape(( a1.shape[0], -1, 32, a1.shape[-1])).transpose(2, 3)
 a1 = self.pre_block4(a1, pool_size=(2, 1))
x = self.spectrogram_extractor(input)
 x = self.logmel_extractor(x)
 x = x.transpose(1, 3)
 x = self.bn0(x)
 x = x.transpose(1, 3)
 if self.training:
x = self.spec_augmenter(x)
 if self.training and mixup_lambda is not None:
x = do_mixup(x, mixup_lambda)
 a1 = do_mixup(a1, mixup_lambda)
 x = self.conv_block1(x, pool_size=(2, 2), pool_type='avg')
 x = torch.cat(( x, a1), dim=1)
 x = F.dropout(x, p=0.2, training=self.training)
 x = self.conv_block2(x, pool_size=(2, 2), pool_type='avg')
 x = F.dropout(x, p=0.2, training=self.training)
 x = self.conv_block3(x, pool_size=(2, 2), pool_type='avg')
 x = F.dropout(x, p=0.2, training=self.training)
 x = self.conv_block4(x, pool_size=(2, 2), pool_type='avg')
 x = F.dropout(x, p=0.2, training=self.training)
 x = self.conv_block5(x, pool_size=(2, 2), pool_type='avg')
 x = F.dropout(x, p=0.2, training=self.training)
 x = self.conv_block6(x, pool_size=(1, 1), pool_type='avg')
 x = F.dropout(x, p=0.2, training=self.training)
 x = torch.mean(x, dim=3)
(x1, _)= torch.max(x, dim=2)
 x2 = torch.mean(x, dim=2)
 x = x1 + x2
x = F.dropout(x, p=0.5, training=self.training)
 x = F.relu_(self.fc1(x))
embedding = F.dropout(x, p=0.5, training=self.training)
 clipwise_output = torch.sigmoid(self.fc_audioset(x))
output_dict = {'clipwise_output': clipwise_output, 'embedding': embedding}
return output_dict
GENES = [col for col in train.columns if col.startswith('g-')]
CELLS = [col for col in train.columns if col.startswith('c-')]<import_modules>
def get_model(weight_path=None):
model_config = {
"sample_rate": 32000,
"window_size": 1024,
"hop_size": 320,
"mel_bins": 64,
"fmin": 50,
"fmax": 14000,
"classes_num":264
}
model = Wavegram_Logmel_Cnn14(**model_config)
 if weight_path:
print("load pretrain weight: {}".format(weight_path))
weights = torch.load(weight_path, map_location=torch.device('cpu'))
model.load_state_dict(weights['model_state_dict'])
 model.cuda()
model.eval()
return model
from sklearn.decomposition import PCA
from sklearn.preprocessing import QuantileTransformer<import_modules>
def prediction_for_clip(test_df,
clip,
models,
threshold=0.5):
dataset = TestDataset(df=test_df, clip=clip)
 loader = data.DataLoader(dataset, batch_size=1, shuffle=False)
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 prediction_dict = {}
for image, row_id, site in progress_bar(loader):
site = site[0]
row_id = row_id[0]
image = image.to(device ).float()
if site in {"site_1", "site_2"}:
image = image.to(device ).float()
with torch.no_grad() :
proba = 0.0
for model in models:
model.eval()
prediction = model(image)
 proba += prediction['clipwise_output'].detach().cpu().numpy().reshape(-1)
 proba /= len(models)
 events = proba >= threshold
labels = np.argwhere(events ).reshape(-1 ).tolist()
else:
image = image.squeeze(0)
 batch_size = 16
whole_size = image.size(0)
 if whole_size % batch_size == 0:
n_iter = whole_size // batch_size
else:
n_iter = whole_size // batch_size + 1
all_events = set()
for batch_i in range(n_iter):
batch = image[batch_i * batch_size:(batch_i + 1)* batch_size]
if batch.ndim == 3:
batch = batch.unsqueeze(0)
 batch = batch.to(device)
 with torch.no_grad() :
proba = 0.0
for model in models:
model.eval()
prediction = model(image)
 proba += prediction['clipwise_output'].detach().cpu().numpy()
proba /= len(models)
 events = proba >= threshold
for i in range(len(events)) :
event = events[i, :]
labels = np.argwhere(event ).reshape(-1 ).tolist()
for label in labels:
all_events.add(label)
 labels = list(all_events)
 if len(labels)== 0:
prediction_dict[row_id] = "nocall"
else:
labels_str_list = list(map(lambda x: INV_BIRD_CODE[x], labels))
label_string = " ".join(labels_str_list)
 prediction_dict[row_id] = label_string
return prediction_dict
from sklearn.compose import make_column_transformer,ColumnTransformer<set_options>
def prediction(test_df,
test_audio,
weight_paths,
target_sr,
threshold=0.5):
models = []
for weight_path in weight_paths:
models.append(get_model(weight_path))
unique_audio_id = test_df.audio_id.unique()
warnings.filterwarnings("ignore")
 prediction_dfs = []
for audio_id in unique_audio_id:
with timer(f"Loading {audio_id}"):
clip, _ = librosa.load(test_audio /(audio_id + ".mp3"),
sr=target_sr,
mono=True,
res_type="kaiser_fast")
 test_df_for_audio_id = test_df.query(
 f"audio_id == '{audio_id}'" ).reset_index(drop=True)
 with timer(f"Prediction on {audio_id}"):
prediction_dict = prediction_for_clip(test_df_for_audio_id,
clip=clip,
models=models,
threshold=threshold)
 row_id = list(prediction_dict.keys())
 birds = list(prediction_dict.values())
 prediction_df = pd.DataFrame({
"row_id": row_id,
"birds": birds
})
 prediction_dfs.append(prediction_df)
 prediction_df = pd.concat(prediction_dfs, axis=0, sort=False ).reset_index(drop=True)
 return prediction_df
def seed_everything(seed=42):
random.seed(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(seed=42 )<categorify>
submission = prediction(test_df=test,
test_audio=TEST_AUDIO_DIR,
weight_paths=glob(".. /input/pann-best-weights/*.bin"),
target_sr=32000,
threshold=0.6)
 submission.to_csv("submission.csv", index=False )
<categorify><EOS> 
submission['birds'].value_counts()