Kaggle competition: Cornell Birdcall Identification
<SOS> metric: MeanFScoreBeta Kaggle data source: cornell-birdcall-identification<import_modules>
!pip uninstall torch --y
!pip uninstall torchaudio --y
!pip install.. /input/pytorch-160-with-torchvision-070/torch-1.6.0cu101-cp37-cp37m-linux_x86_64.whl
!pip install.. /input/torchaudio/torchaudio-0.6.0-cp37-cp37m-manylinux1_x86_64.whl
import pandas as pd
import numpy as np<import_modules>
!pip install.. /input/timm-wheel/*.whl
import pandas as pd
import numpy as np<load_from_csv>
warnings.filterwarnings('ignore',".*PySoundFile")
 pd.options.display.max_rows = 500
pd.options.display.max_columns = 500
! python.. /input/better-cv-pl-v1/v1_inference.py
! python.. /input/moa-sin-v2/v2_inference.py
! python.. /input/baseline-inference/moa_lstm_inference.py
! python.. /input/resnet-sin-v3/v3_inference.py
! python.. /input/inference-scripts/moa-tabnet-correct-inference-freezepca.py
! python.. /input/baseline-inference/moa_v6.py
! python.. /input/baseline-inference/moa_v7.py
! python.. /input/baseline-inference/moa_v8.py
! python.. /input/baseline-inference/moa_inference_non_scored.py
! python.. /input/resnet-inference-script/moa-multi-input-resnet-model-inference.py
submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv')
 submission.iloc[:, 1:] = 0
subv1 = pd.read_csv('./submission_v1_sin.csv')
 subv2 = pd.read_csv('./submission_v2_sin.csv')
 subv3 = pd.read_csv('./submission_v3_sin.csv')
 subv4 = pd.read_csv('./submission_v4_sin.csv')
 subv5 = pd.read_csv('./submission_v5_sin.csv')
 subv6 = pd.read_csv('./submission_v6.csv')
 subv7 = pd.read_csv('./submission_v7.csv')
 subv8 = pd.read_csv('./submission_v8.csv')
 subv9 = pd.read_csv('./submission_tabnet_correct.csv')
 subv10 = pd.read_csv('./submission_resnet.csv' )<define_variables>
RAW_DATA = ".. /input/birdsong-recognition/"
TRAIN_AUDIO_DIR = ".. /input/birdsong-recognition/train_audio/"
TEST_AUDIO_DIR = ".. /input/birdsong-recognition/test_audio/"
weight = [0.32507418/3, 0.32528637/3, 0., 0.34963945/3, 1/9, 1/9, 1/9, 1/9, 1/9, 1/9]<count_values>
train = pd.read_csv(RAW_DATA + "train.csv" )
print(sum(weight), len(weight))<define_variables>
if not os.path.exists(TEST_AUDIO_DIR):
print("Run check")
 TEST_AUDIO_DIR = ".. /input/birdcall-check/test_audio/"
test = pd.read_csv(".. /input/birdcall-check/test.csv")
 else:
print("run submission")
 test = pd.read_csv(".. /input/birdsong-recognition/test.csv" )
subs = [subv1, subv2, subv3, subv4, subv5, subv6, subv7, subv8, subv9, subv10]<feature_engineering>
BIRD_CODE = {
'aldfly': 0, 'ameavo': 1, 'amebit': 2, 'amecro': 3, 'amegfi': 4,
'amekes': 5, 'amepip': 6, 'amered': 7, 'amerob': 8, 'amewig': 9,
'amewoo': 10, 'amtspa': 11, 'annhum': 12, 'astfly': 13, 'baisan': 14,
'baleag': 15, 'balori': 16, 'banswa': 17, 'barswa': 18, 'bawwar': 19,
'belkin1': 20, 'belspa2': 21, 'bewwre': 22, 'bkbcuc': 23, 'bkbmag1': 24,
'bkbwar': 25, 'bkcchi': 26, 'bkchum': 27, 'bkhgro': 28, 'bkpwar': 29,
'bktspa': 30, 'blkpho': 31, 'blugrb1': 32, 'blujay': 33, 'bnhcow': 34,
'boboli': 35, 'bongul': 36, 'brdowl': 37, 'brebla': 38, 'brespa': 39,
'brncre': 40, 'brnthr': 41, 'brthum': 42, 'brwhaw': 43, 'btbwar': 44,
'btnwar': 45, 'btywar': 46, 'buffle': 47, 'buggna': 48, 'buhvir': 49,
'bulori': 50, 'bushti': 51, 'buwtea': 52, 'buwwar': 53, 'cacwre': 54,
'calgul': 55, 'calqua': 56, 'camwar': 57, 'cangoo': 58, 'canwar': 59,
'canwre': 60, 'carwre': 61, 'casfin': 62, 'caster1': 63, 'casvir': 64,
'cedwax': 65, 'chispa': 66, 'chiswi': 67, 'chswar': 68, 'chukar': 69,
'clanut': 70, 'cliswa': 71, 'comgol': 72, 'comgra': 73, 'comloo': 74,
'commer': 75, 'comnig': 76, 'comrav': 77, 'comred': 78, 'comter': 79,
'comyel': 80, 'coohaw': 81, 'coshum': 82, 'cowscj1': 83, 'daejun': 84,
'doccor': 85, 'dowwoo': 86, 'dusfly': 87, 'eargre': 88, 'easblu': 89,
'easkin': 90, 'easmea': 91, 'easpho': 92, 'eastow': 93, 'eawpew': 94,
'eucdov': 95, 'eursta': 96, 'evegro': 97, 'fiespa': 98, 'fiscro': 99,
'foxspa': 100, 'gadwal': 101, 'gcrfin': 102, 'gnttow': 103, 'gnwtea': 104,
'gockin': 105, 'gocspa': 106, 'goleag': 107, 'grbher3': 108, 'grcfly': 109,
'greegr': 110, 'greroa': 111, 'greyel': 112, 'grhowl': 113, 'grnher': 114,
'grtgra': 115, 'grycat': 116, 'gryfly': 117, 'haiwoo': 118, 'hamfly': 119,
'hergul': 120, 'herthr': 121, 'hoomer': 122, 'hoowar': 123, 'horgre': 124,
'horlar': 125, 'houfin': 126, 'houspa': 127, 'houwre': 128, 'indbun': 129,
'juntit1': 130, 'killde': 131, 'labwoo': 132, 'larspa': 133, 'lazbun': 134,
'leabit': 135, 'leafly': 136, 'leasan': 137, 'lecthr': 138, 'lesgol': 139,
'lesnig': 140, 'lesyel': 141, 'lewwoo': 142, 'linspa': 143, 'lobcur': 144,
'lobdow': 145, 'logshr': 146, 'lotduc': 147, 'louwat': 148, 'macwar': 149,
'magwar': 150, 'mallar3': 151, 'marwre': 152, 'merlin': 153, 'moublu': 154,
'mouchi': 155, 'moudov': 156, 'norcar': 157, 'norfli': 158, 'norhar2': 159,
'normoc': 160, 'norpar': 161, 'norpin': 162, 'norsho': 163, 'norwat': 164,
'nrwswa': 165, 'nutwoo': 166, 'olsfly': 167, 'orcwar': 168, 'osprey': 169,
'ovenbi1': 170, 'palwar': 171, 'pasfly': 172, 'pecsan': 173, 'perfal': 174,
'phaino': 175, 'pibgre': 176, 'pilwoo': 177, 'pingro': 178, 'pinjay': 179,
'pinsis': 180, 'pinwar': 181, 'plsvir': 182, 'prawar': 183, 'purfin': 184,
'pygnut': 185, 'rebmer': 186, 'rebnut': 187, 'rebsap': 188, 'rebwoo': 189,
'redcro': 190, 'redhea': 191, 'reevir1': 192, 'renpha': 193, 'reshaw': 194,
'rethaw': 195, 'rewbla': 196, 'ribgul': 197, 'rinduc': 198, 'robgro': 199,
'rocpig': 200, 'rocwre': 201, 'rthhum': 202, 'ruckin': 203, 'rudduc': 204,
'rufgro': 205, 'rufhum': 206, 'rusbla': 207, 'sagspa1': 208, 'sagthr': 209,
'savspa': 210, 'saypho': 211, 'scatan': 212, 'scoori': 213, 'semplo': 214,
'semsan': 215, 'sheowl': 216, 'shshaw': 217, 'snobun': 218, 'snogoo': 219,
'solsan': 220, 'sonspa': 221, 'sora': 222, 'sposan': 223, 'spotow': 224,
'stejay': 225, 'swahaw': 226, 'swaspa': 227, 'swathr': 228, 'treswa': 229,
'truswa': 230, 'tuftit': 231, 'tunswa': 232, 'veery': 233, 'vesspa': 234,
'vigswa': 235, 'warvir': 236, 'wesblu': 237, 'wesgre': 238, 'weskin': 239,
'wesmea': 240, 'wessan': 241, 'westan': 242, 'wewpew': 243, 'whbnut': 244,
'whcspa': 245, 'whfibi': 246, 'whtspa': 247, 'whtswi': 248, 'wilfly': 249,
'wilsni1': 250, 'wiltur': 251, 'winwre3': 252, 'wlswar': 253, 'wooduc': 254,
'wooscj2': 255, 'woothr': 256, 'y00475': 257, 'yebfly': 258, 'yebsap': 259,
'yehbla': 260, 'yelwar': 261, 'yerwar': 262, 'yetvir': 263,'nocall':-1,
}
INV_BIRD_CODE = {v: k for k, v in BIRD_CODE.items() }
for w, sub in zip(weight, subs):
submission.iloc[:, 1:] += sub.iloc[:,1:] * w<save_to_csv>
class Dataset_Test:
def __init__(
 self,
df,
data_dir,
sample_rate=32000,
duration=5,
waveform_transforms=None):
self.data_dir=data_dir
self.clip_frames=sample_rate*duration
self.df=df
self.sample_rate=sample_rate
self.duration=duration
self.audio_ids=df.audio_id.unique()
def __len__(self):
return len(self.audio_ids)
 def __getitem__(self, idx: int):
audio_id=self.audio_ids[idx]
clip, _ = librosa.load(self.data_dir+'{}.mp3'.format(audio_id),
sr=self.sample_rate,
mono=True,
res_type="kaiser_fast")
 clip=clip.astype(np.float32)
 total_frames=len(clip)
 target_df=self.df[self.df.audio_id==audio_id]
all_5s_clips=[]
row_ids=[]
site=target_df.site.iloc[0]
if site=='site_3':
startpoints=list(range(0,total_frames,self.clip_frames))
if(len(startpoints)>1 and total_frames-startpoints[-1]<= 2.5*self.sample_rate):
startpoints.pop(-1)
 for start in startpoints:
clip_5s=clip[start:min(total_frames,start+self.clip_frames)]
if len(clip_5s)<self.clip_frames:
pad0=(self.clip_frames-len(clip_5s)) //2
pad1=self.clip_frames-len(clip_5s)-pad0
clip_5s=np.pad(clip_5s,[pad0,pad1],constant_values=0)
 all_5s_clips.append(clip_5s)
 row_ids.append("site_3_{}".format(audio_id))
else:
for i in range(len(target_df)) :
row=target_df.iloc[i]
end=int(row.seconds*self.sample_rate)
 clip_5s=clip[max(0,end-self.clip_frames):min(total_frames,end)]
if len(clip_5s)<self.clip_frames:
pad0=(self.clip_frames-len(clip_5s)) //2
pad1=self.clip_frames-len(clip_5s)-pad0
clip_5s=np.pad(clip_5s,[pad0,pad1],constant_values=0)
 all_5s_clips.append(clip_5s)
 row_ids.append("{}_{}_{}".format(site,audio_id,int(row.seconds)))
 all_5s_clips=np.stack(all_5s_clips,axis=0)
 return torch.tensor(all_5s_clips),row_ids,site
submission.to_csv('submission.csv', index=False )<save_to_csv>
def gem(x, p=3, eps=1e-6):
return F.avg_pool2d(x.clamp(min=eps ).pow(p),(x.size(-2), x.size(-1)) ).pow(1./ p)
 class GeM(nn.Module):
def __init__(self, p=3, eps=1e-6):
super(GeM, self ).__init__()
self.p = nn.Parameter(torch.ones(1)* p)
 self.eps = eps
def forward(self, x):
return gem(x, p=self.p, eps=self.eps)
 def __repr__(self):
return self.__class__.__name__ + '(' + 'p=' + '{:.4f}'.format(self.p.data.tolist() [0])+ ', ' + 'eps=' + str(
 self.eps)+ ')'
class AdaptiveConcatPool2d(nn.Module):
def __init__(self, sz=None,gem=False):
super().__init__()
sz = sz or(1,1)
 self.ap = GeM() if gem else nn.AdaptiveAvgPool2d(sz)
 self.mp = nn.AdaptiveMaxPool2d(sz)
 def forward(self, x):
return torch.cat([self.ap(x), self.mp(x)], 1 ).view(x.size(0),-1)
 class AttMaxPool(nn.Module):

 def __init__(self, n_in,hidden=512,dropout=0,drop_time=0.) :
super().__init__()
att_module = [
nn.Conv1d(n_in, hidden, kernel_size=1, stride=1, padding=0, bias=True),
nn.Tanh() ,
]
if dropout>0:
att_module.append(nn.Dropout(dropout))
att_module.append(nn.Conv1d(hidden,1,kernel_size=1,stride=1,padding=0,bias=True))
self.att=nn.Sequential(*att_module)
 self.drop_time=drop_time
def forward(self, x):
mp=torch.max(x,dim=2)[0]
A=self.att(x)
 A=torch.softmax(A,dim=-1)
 if self.drop_time>0 and self.training:
A=nn.functional.dropout(A,p=self.drop_time,training=self.training)
 A/=A.sum(dim=2,keepdim=True)
 print("drop")
 x = torch.sum(A * x, dim=2)
 x = torch.cat([x,mp],dim=1)
 return x
class ModelWarper(nn.Module):
def __init__(self,arch,dropout=0.5,num_class=264):
super().__init__()
self.basemodel=timm.create_model(arch,pretrained=False)
 features=self.basemodel.get_classifier().in_features
self.pooling=AdaptiveConcatPool2d()
self.dropout=nn.Dropout(dropout)
 self.fc=nn.Linear(2*features,out_features=num_class)
 def forward(self,x):
x=self.basemodel.forward_features(x)
 x=self.pooling(x)
 x=x.view(x.size(0),-1)
 x=self.dropout(x)
 x=self.fc(x)
 return x
class ModelWarper_Att(nn.Module):
def __init__(self,arch,dropout=0.5,dropout_att=0.25,num_class=264,drop_time=0,**kwargs):
super().__init__()
if arch=='PANN_res54':
self.basemodel=ResNet54()
ckpt=torch.load("./model/ResNet54.pth")
 self.basemodel.load_state_dict(ckpt['model'],strict=False)
 features=2048
else:
self.basemodel=timm.create_model(arch,pretrained=False,**kwargs)
 features=self.basemodel.get_classifier().in_features
self.pooling=AttMaxPool(features,dropout=dropout_att,drop_time=drop_time)
 self.dropout=nn.Dropout(dropout)
 self.fc=nn.Linear(2*features,out_features=num_class)
 def forward(self,x):
x=self.basemodel.forward_features(x)
 x=x.mean(dim=2)
 x=self.pooling(x)
 x=self.dropout(x)
 x=self.fc(x)
 return x
submission.to_csv('submission.csv', index=False )<import_modules>
preprocessor=dict()
mel_config = {
"sample_rate": 32000,
"n_fft": [1024,1024,1024],
"hop_length": 313,
"n_mels": 128,
"f_min": 20,
"f_max": 16000,
}
transformer = MelTransformer(**mel_config ).cuda()
augmenter=SpecAugmenter(valid=True,resize=256,crop_height=None)
 preprocessor['standard']={
'transformer':transformer,
'augmenter':augmenter
}
mel_config_2 = {
"sample_rate": 32000,
"n_fft": [1024,1024,1024],
"hop_length": 209,
"n_mels": 128,
"f_min": 20,
"f_max": 16000,
}
transformer = MelTransformer(**mel_config_2 ).cuda()
augmenter=SpecAugmenter(valid=True,resize=320,crop_height=None)
 preprocessor['big']={
'transformer':transformer,
'augmenter':augmenter
}
mel_config_3 = {
"sample_rate": 32000,
"n_fft": [1024,1024,1024],
"hop_length": 313,
"n_mels": 96,
"f_min": 20,
"f_max": 16000,
}
transformer = MelTransformer(**mel_config_3 ).cuda()
augmenter=SpecAugmenter(valid=True,resize=224,crop_height=None)
 preprocessor['small']={
'transformer':transformer,
'augmenter':augmenter
}
import pickle
import numpy as np
import pandas as pd
import tensorflow as tf<choose_model_class>
model_configs=[
{
'arch':ModelWarper,
'preprocess':'standard',
'path':".. /input/bird-resnest50-mp3ft-5fold/resnest50d_1s4x24d_fold0_epoch9.pth",
'base':"resnest50d_1s4x24d",
},
{
'arch':ModelWarper,
'preprocess':'standard',
'path':".. /input/bird-resnest50-mp3ft-5fold/resnest50d_1s4x24d_fold1_epoch9.pth",
'base':"resnest50d_1s4x24d",
},
{
'arch':ModelWarper,
'preprocess':'standard',
'path':".. /input/bird-resnest50-mp3ft-5fold/resnest50d_1s4x24d_fold2_epoch9.pth",
'base':"resnest50d_1s4x24d",
},
{
'arch':ModelWarper,
'preprocess':'standard',
'path':".. /input/bird-resnest50-mp3ft-5fold/resnest50d_1s4x24d_fold3_epoch9.pth",
'base':"resnest50d_1s4x24d",
},
{
'arch':ModelWarper,
'preprocess':'standard',
'path':".. /input/bird-resnest50-mp3ft-5fold/resnest50d_1s4x24d_fold4_epoch9.pth",
'base':"resnest50d_1s4x24d",
},
{
'arch':ModelWarper,
'preprocess':'big',
'path':".. /input/birdcall-bigsize/resnest50d_1s4x24d_fold0_epoch9.pth",
'base':"resnest50d_1s4x24d",
},
{
'arch':ModelWarper,
'preprocess':'big',
'path':".. /input/birdcall-bigsize/resnest50d_1s4x24d_fold1_epoch9.pth",
'base':"resnest50d_1s4x24d",
},
{
'arch':ModelWarper,
'preprocess':'big',
'path':".. /input/birdcall-bigsize/resnest50d_1s4x24d_fold2_epoch9.pth",
'base':"resnest50d_1s4x24d",
},
{
'arch':ModelWarper,
'preprocess':'big',
'path':".. /input/birdcall-bigsize/resnest50d_1s4x24d_fold3_epoch9.pth",
'base':"resnest50d_1s4x24d",
},
{
'arch':ModelWarper,
'preprocess':'big',
'path':".. /input/birdcall-bigsize/resnest50d_1s4x24d_fold4_epoch9.pth",
'base':"resnest50d_1s4x24d",
},
{
'arch':ModelWarper,
'preprocess':'small',
'path':".. /input/regnety40-224/regnety_040_fold0_epoch9.pth",
'base':"regnety_040",
},
{
'arch':ModelWarper,
'preprocess':'small',
'path':".. /input/regnety40-224/regnety_040_fold1_epoch9.pth",
'base':"regnety_040",
},
{
'arch':ModelWarper,
'preprocess':'small',
'path':".. /input/regnety40-224/regnety_040_fold2_epoch9.pth",
'base':"regnety_040",
},
{
'arch':ModelWarper,
'preprocess':'small',
'path':".. /input/regnety40-224/regnety_040_fold3_epoch9.pth",
'base':"regnety_040",
},
]
def build_callbacks(
 model_path, factor=0.1, mode="auto", monitor="val_loss", patience=0, verbose=0 
):
early_stopping = tf.keras.callbacks.EarlyStopping(
 mode=mode, monitor=monitor, patience=patience, verbose=verbose 
)
 model_checkpoint = tf.keras.callbacks.ModelCheckpoint(
 model_path, mode=mode, monitor=monitor, save_best_only=True, verbose=verbose 
)
 reduce_lr_on_plateau = tf.keras.callbacks.ReduceLROnPlateau(
 factor=factor, monitor=monitor, mode=mode, verbose=verbose 
)
 return [early_stopping, model_checkpoint, reduce_lr_on_plateau]<init_hyperparams>
modellist=[]
pplist=[]
for config in model_configs:
model=config['arch'](config['base'],num_class=264)
 print("Loading",config['path'])
 ckpt=torch.load(config['path'])['state_dict']
model.load_state_dict(ckpt)
 model.cuda()
model.eval()
modellist.append(model)
 pplist.append(config['preprocess'])
 print(len(modellist))
print(pplist )
def build_mlp_classifier(
 input_dim,
output_dim,
activation="relu",
bias_initializer="zeros",
kernel_initializer="glorot_uniform",
label_smoothing=0.0,
lr=1e-03,
momentum=0.99,
n_layers=3,
n_units="auto",
pretrained_model_path=None,
rate=0.0,
skip=False, 
):
if n_units == "auto":
n_units = 0.5 *(input_dim + output_dim)
 inputs = tf.keras.layers.Input(shape=input_dim, name="I")
 x = inputs
for i in range(n_layers - 2):
x = tfa.layers.WeightNormalization(
 tf.keras.layers.Dense(n_units, kernel_initializer=kernel_initializer),
name=f"WN{i + 1}",
)(x)
 x = tf.keras.layers.BatchNormalization(momentum=momentum, name=f"BN{i + 1}" )(x)
 if skip and i > 0 and i % 2 == 0:
x = x + shortcut
x = tf.keras.layers.Activation(activation, name=f"A{i + 1}" )(x)
 x = tf.keras.layers.Dropout(rate, name=f"D{i + 1}" )(x)
 if skip and i % 2 == 0:
shortcut = x
x = tfa.layers.WeightNormalization(
 tf.keras.layers.Dense(output_dim, bias_initializer=bias_initializer)
 )(x)
 outputs = tf.keras.layers.Activation("sigmoid" )(x)
 model = tf.keras.models.Model(inputs=inputs, outputs=outputs)
 if pretrained_model_path is not None:
model.load_weights(pretrained_model_path, by_name=True)
 loss = tf.keras.losses.BinaryCrossentropy(label_smoothing=label_smoothing)
 optimizer = AdaBeliefOptimizer(learning_rate=lr)
 model.compile(loss=loss, optimizer=optimizer)
 return model<create_dataframe>
torch.cuda.empty_cache()
def compute_row_statistics(X, prefix=""):
Xt = pd.DataFrame()
for agg_func in [
"mean",
"std",
"kurtosis",
"skew",
]:
Xt[f"{prefix}{agg_func}"] = X.agg(agg_func, axis=1)
 return Xt<compute_test_metric>
!nvidia-smi
<compute_test_metric>
dataset=Dataset_Test(
 test,
TEST_AUDIO_DIR,
duration=5,
sample_rate=32000, 
)
 dataloader=DataLoader(dataset,batch_size=1,shuffle=False,num_workers=2,drop_last=False,pin_memory=True )
def score(Y, Y_pred, eps=1e-15, label_smoothing=0.0):
Y = np.asarray(Y)
 Y = np.ravel(Y)
 if label_smoothing > 0.0:
Y = Y *(1.0 - label_smoothing)+ 0.5 * label_smoothing
Y_pred = np.asarray(Y_pred)
 Y_pred = np.ravel(Y_pred)
 Y_pred = np.clip(Y_pred, eps, 1.0 - eps)
 return -np.mean(Y * np.log(Y_pred)+(1.0 - Y)* np.log(1.0 - Y_pred))<set_options>
THRESHOLD=0.5
MAX_PRED={
'site_1':3,
'site_2':3,
'site_3':10
}
def set_seed(seed=0):
os.environ["PYTHONHASHSEED"] = str(seed)
 rn.seed(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 graph = tf.compat.v1.get_default_graph()
session_conf = tf.compat.v1.ConfigProto(
 inter_op_parallelism_threads=1, intra_op_parallelism_threads=1 
)
 sess = tf.compat.v1.Session(graph=graph, config=session_conf)
 tf.compat.v1.keras.backend.set_session(sess )<train_model>
rows=[]
result=[]
for clips,row_ids,site in tqdm(dataloader):
clips=clips.cuda() [0]
batch_output=0.
 specs={}
specs_shift1={}
specs_shift2={}
shifted1=torch.zeros(clips.shape ).float().cuda()
shifted1[:,:-155]=clips[:,155:]
shifted2=torch.zeros(clips.shape ).float().cuda()
shifted2[:,155:]=clips[:,:-155]
with torch.no_grad() :
for k,v in preprocessor.items() :
spec=v['transformer'](clips)
 spec=v['augmenter'](spec)
 specs[k]=spec
specs_shift1[k]=v['augmenter'](v['transformer'](shifted1))
specs_shift2[k]=v['augmenter'](v['transformer'](shifted2))
for model,p in zip(modellist,pplist):
output = torch.sigmoid(model(specs[p]))
output_shift1=torch.sigmoid(model(specs_shift1[p]))
output_shift2=torch.sigmoid(model(specs_shift2[p]))
batch_output+=(( output+output_shift1+output_shift2)/3)**0.5
batch_output/=len(modellist)
 batch_output = batch_output.cpu().numpy()
if site[0]=='site_3':
batch_output=np.max(batch_output,axis=0,keepdims=True)
 print(batch_output[:,0])
 for clip_pred in batch_output:
binary_pred=clip_pred>THRESHOLD
pos_idx=np.where(binary_pred)[0]
pos_prob=clip_pred[pos_idx]
sort=np.argsort(pos_prob)[::-1]
pos_idx=pos_idx[sort[:min(len(sort),MAX_PRED[site[0]])]]
if len(pos_idx)==0:
result.append("nocall")
 else:
result.append(" ".join([INV_BIRD_CODE[code] for code in pos_idx]))
for r in row_ids:
rows.extend(r)
class ClippedFeatures(BaseEstimator, TransformerMixin):
def __init__(self, copy=True, high=0.99, low=0.01):
self.copy = copy
self.high = high
self.low = low
def fit(self, X, y=None):
self.data_max_ = X.quantile(q=self.high)
 self.data_min_ = X.quantile(q=self.low)
 return self
def transform(self, X):
if self.copy:
X = X.copy()
X.clip(self.data_min_, self.data_max_, axis=1, inplace=True)
 return X<define_variables>
submission_df=pd.DataFrame(
 {
"row_id": rows,
"birds": result
}
)
<init_hyperparams><EOS> 
submission_df.to_csv("submission.csv",index=False )