Kaggle competition: Cornell Birdcall Identification
<SOS> metric: MeanFScoreBeta Kaggle data source: cornell-birdcall-identification<train_model>
pd.options.display.max_rows = 500
pd.options.display.max_columns = 500
def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):
model.train()
final_loss = 0
for data in dataloader:
optimizer.zero_grad()
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
if not scheduler.__class__ == torch.optim.lr_scheduler.ReduceLROnPlateau:
scheduler.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 return final_loss
def valid_fn(model, scheduler, loss_fn, dataloader, device):
model.eval()
final_loss = 0
valid_preds = []
for data in dataloader:
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 if scheduler.__class__ == torch.optim.lr_scheduler.ReduceLROnPlateau:
scheduler.step(final_loss)
 return final_loss, valid_preds
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
inputs = data['x'].to(device)
 with torch.no_grad() :
outputs = model(inputs)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds
def infer_features_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
inputs = data['x'].to(device)
 with torch.no_grad() :
outputs = model(inputs,mode='get_features')
 preds.append(torch.cat(outputs,axis=1 ).detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds 
<load_from_csv>
def set_seed(seed: int = 42):
random.seed(seed)
 np.random.seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 @contextmanager
def timer(name: str)-> None:

 t0 = time.time()
print("[{}] start".format(name))
yield
print("[{}] done in {:.0f} s".format(name, time.time() - t0))
SEED = [0]
train,target_cols = train_short_form_loader('.. /input/lish-moa/train_features.csv','.. /input/lish-moa/train_targets_scored.csv')
 test = test_short_form_loader(".. /input/lish-moa/test_features.csv")
 train = pd.concat([train,test])
 train[target_cols]= train[target_cols].fillna(0)
 test = train.copy()
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for seed in SEED:
train = multifold_indexer(train,target_cols,n_splits=NFOLDS)
 for fold in range(NFOLDS):
train_df = train[train['kfold'] != fold]
valid_df = train[train['kfold'] == fold]
feature_cols = [col for col in train_df.columns if not(col in target_cols+['kfold'])]
pipeline_val = make_pipeline(*transformers_list)
 X_train, y_train = train_df[feature_cols], train_df[target_cols]
X_valid, y_valid = valid_df[feature_cols], valid_df[target_cols].values
pipeline_val = load(BOOK2+'pipeline_val2_fold%i.joblib'%fold)
 print('LOADED PIPELINE_VAL2_FOLD%i'%fold)
 print(pipeline_val)
 X_train = pipeline_val.transform(X_train)
 X_train = X_train.values
X_valid = pipeline_val.transform(X_valid)
 valid_index = X_valid.index
X_valid = X_valid.values
y_train = y_train.values
X_test = test[feature_cols]
X_test = pipeline_val.transform(X_test ).values
pred_ = run_inference(X_train,y_train,X_valid,y_valid,X_test,fold, seed,inference_only=True)
 print('
transformed_features = pd.DataFrame(pred_[1],index=test.index)
 transformed_features.columns = [str(i)for i in range(len(transformed_features.columns)) ]
transformed_features.reset_index().to_feather('./features_0.2_altogether_fold%i.fth'%fold)
 if fold>=(CTT-1): break<import_modules>
set_seed(1213 )
from sklearn.base import BaseEstimator, TransformerMixin<categorify>
ROOT = Path.cwd().parent
INPUT_ROOT = ROOT / "input"
RAW_DATA = INPUT_ROOT / "birdsong-recognition"
TRAIN_AUDIO_DIR = RAW_DATA / "train_audio"
TEST_AUDIO_DIR = RAW_DATA / "test_audio"
class MoADataset:
def __init__(self, features, targets):
self.features = features
self.targets = targets
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float),
'y' : torch.tensor(self.targets[idx, :], dtype=torch.float)
 }
return dct
class TestDataset:
def __init__(self, features):
self.features = features
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float)
 }
return dct 
<set_options>
train = pd.read_csv(RAW_DATA / "train.csv" )
def seed_everything(seed=42):
random.seed(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True 
<categorify>
if not TEST_AUDIO_DIR.exists() :
TEST_AUDIO_DIR = INPUT_ROOT / "birdcall-check" / "test_audio"
test = pd.read_csv(INPUT_ROOT / "birdcall-check" / "test.csv")
 else:
test = pd.read_csv(RAW_DATA / "test.csv" )
class DaeAdder(BaseEstimator, TransformerMixin):
def __init__(self,filename):
self.filename=filename
def fit(self,X,y=None):
return self
def fit_transform(self,X,y=None):
Dae_features = pd.read_feather(self.filename ).set_index('sig_id')
 return X.merge(Dae_features,how='left', on='sig_id')
 def transform(self,X):
Dae_features = pd.read_feather(self.filename ).set_index('sig_id')
 return X.merge(Dae_features,how='left', on='sig_id')
<train_model>
sub = pd.read_csv(".. /input/birdsong-recognition/sample_submission.csv")
 sub.to_csv("submission.csv", index=False )
class SupressControls(BaseEstimator, TransformerMixin):
def __init__(self):
pass
def fit(self, X, y = None):
return self
def fit_transform(self, X, y = None):
X = X[X['cp_type']!=0]
X = X.drop('cp_type', axis=1)
 return X
def transform(self, X):
X = X[X['cp_type']!=0]
X = X.drop('cp_type', axis=1)
 return X
class CatIntMapper(BaseEstimator, TransformerMixin):
def __init__(self ,col,dicti):
self.col = col
self.dicti = dicti
def fit(self, X, y = None):
return self
def fit_transform(self, X, y = None):
assert X[self.col].isin(self.dicti.keys() ).all()
return pd.concat([X.drop(self.col,axis=1),X[self.col].map(self.dicti ).astype(int ).rename(self.col)],axis=1)
 def transform(self, X):
assert X[self.col].isin(self.dicti.keys() ).all()
return pd.concat([X.drop(self.col,axis=1),X[self.col].map(self.dicti ).astype(int ).rename(self.col)],axis=1)
 class NamedOutTWrapper(BaseEstimator, TransformerMixin):
def __init__(self,transformer,columns,inplace=False,prefix='_'):
self.transformer = transformer
self.cols = columns
self.inplace = inplace
self.prefix = prefix
self.transformer_name = self._get_transformer_name()
def fit(self, X, y = None):
self.transformer = self.transformer.fit(X[self.cols] , y)
 return self
def fit_transform(self, X, y = None):
transformed_columns = self.transformer.fit_transform(X[self.cols] , y)
 out=pd.DataFrame(index=X.index)
 if self.inplace:
out = X[self.cols].copy()
out[self.cols] = transformed_columns
return pd.concat([X.drop(self.cols,axis=1),out],axis=1)
 else:
for i,values in enumerate(transformed_columns.transpose()):
out[ self.transformer_name + self.prefix + str(i)] = values
return pd.concat([X,out],axis=1)
 def transform(self, X):
transformed_columns = self.transformer.transform(X[self.cols])
 out=pd.DataFrame(index=X.index)
 if self.inplace:
out = X[self.cols].copy()
out[self.cols] = transformed_columns
return pd.concat([X.drop(self.cols,axis=1),out],axis=1)
 else:
for i,values in enumerate(transformed_columns.transpose()):
out[ self.transformer_name + self.prefix + str(i)] = values
return pd.concat([X,out],axis=1)
 def _get_transformer_name(self):
return str(self.transformer.__class__ ).split('.')[-1][0:-2]
class IdentityTransformer:

 def __init__(self):
pass
def fit(self, X, y = None):
return self
def fit_transform(self, X, y = None):
return X
def transform(self, X):
return X
class SuppressControls(BaseEstimator, TransformerMixin):
def __init__(self):
pass
def fit(self, X, y = None):
return self
def fit_transform(self, X, y = None):
return X.loc[X['cp_type']=='trt_cp'].drop('cp_type', axis=1)
 def transform(self, X):
return X.loc[X['cp_type']=='trt_cp'].drop('cp_type', axis=1)
 class ColumnDropper(BaseEstimator, TransformerMixin):
def __init__(self, cols):
self.cols=cols
def fit(self, X, y = None):
return self
def transform(self, X, y = None):
return X.drop(self.cols,axis=1)
 class VarianceFilter(BaseEstimator, TransformerMixin):
def __init__(self,threshold):
self.threshold = threshold
def fit(self,X,y=None):
mask = X.var() <= self.threshold
self.drop_cols = set([ col for val,col in zip(mask,X.columns)if val])
 self.drop_cols.discard('cp_type')
 return self
def transform(self,X):
return X.drop(self.drop_cols,axis=1)
 def apply_pipe_together(pipeline,train,test,ct=None):
data = pd.concat([train,test])
 pipeline = load(BOOK2+'pipe_together_ct%i.joblib'%ct)
 print('LOAD PIPE_TOGETHER_CT%i'%ct)
 print(pipeline)
 data = pipeline.transform(data)
 train = data.loc[data.index.intersection(train.index)]
test = data.loc[data.index.intersection(test.index)]
return pipeline,train,test<categorify>
TARGET_SR = 32000
model_config = {
"base_model_name": "resnest50_fast_1s1x64d",
"pretrained": False,
"num_classes": 264,
"trained_weights": ".. /input/training-birdsong-baseline-resnest50-fast/best_model.pth"
}
melspectrogram_parameters = {
"n_mels": 128,
"fmin": 20,
"fmax": 16000
}
map_controls = CatIntMapper('cp_type',{'ctl_vehicle': 0, 'trt_cp': 1})
 map_dose = CatIntMapper('cp_dose',{'D1': 1, 'D2': 0})
 map_time = CatIntMapper('cp_time',{24: 0, 48: 1, 72: 2})
<normalization>
BIRD_CODE = {
'aldfly': 0, 'ameavo': 1, 'amebit': 2, 'amecro': 3, 'amegfi': 4,
'amekes': 5, 'amepip': 6, 'amered': 7, 'amerob': 8, 'amewig': 9,
'amewoo': 10, 'amtspa': 11, 'annhum': 12, 'astfly': 13, 'baisan': 14,
'baleag': 15, 'balori': 16, 'banswa': 17, 'barswa': 18, 'bawwar': 19,
'belkin1': 20, 'belspa2': 21, 'bewwre': 22, 'bkbcuc': 23, 'bkbmag1': 24,
'bkbwar': 25, 'bkcchi': 26, 'bkchum': 27, 'bkhgro': 28, 'bkpwar': 29,
'bktspa': 30, 'blkpho': 31, 'blugrb1': 32, 'blujay': 33, 'bnhcow': 34,
'boboli': 35, 'bongul': 36, 'brdowl': 37, 'brebla': 38, 'brespa': 39,
'brncre': 40, 'brnthr': 41, 'brthum': 42, 'brwhaw': 43, 'btbwar': 44,
'btnwar': 45, 'btywar': 46, 'buffle': 47, 'buggna': 48, 'buhvir': 49,
'bulori': 50, 'bushti': 51, 'buwtea': 52, 'buwwar': 53, 'cacwre': 54,
'calgul': 55, 'calqua': 56, 'camwar': 57, 'cangoo': 58, 'canwar': 59,
'canwre': 60, 'carwre': 61, 'casfin': 62, 'caster1': 63, 'casvir': 64,
'cedwax': 65, 'chispa': 66, 'chiswi': 67, 'chswar': 68, 'chukar': 69,
'clanut': 70, 'cliswa': 71, 'comgol': 72, 'comgra': 73, 'comloo': 74,
'commer': 75, 'comnig': 76, 'comrav': 77, 'comred': 78, 'comter': 79,
'comyel': 80, 'coohaw': 81, 'coshum': 82, 'cowscj1': 83, 'daejun': 84,
'doccor': 85, 'dowwoo': 86, 'dusfly': 87, 'eargre': 88, 'easblu': 89,
'easkin': 90, 'easmea': 91, 'easpho': 92, 'eastow': 93, 'eawpew': 94,
'eucdov': 95, 'eursta': 96, 'evegro': 97, 'fiespa': 98, 'fiscro': 99,
'foxspa': 100, 'gadwal': 101, 'gcrfin': 102, 'gnttow': 103, 'gnwtea': 104,
'gockin': 105, 'gocspa': 106, 'goleag': 107, 'grbher3': 108, 'grcfly': 109,
'greegr': 110, 'greroa': 111, 'greyel': 112, 'grhowl': 113, 'grnher': 114,
'grtgra': 115, 'grycat': 116, 'gryfly': 117, 'haiwoo': 118, 'hamfly': 119,
'hergul': 120, 'herthr': 121, 'hoomer': 122, 'hoowar': 123, 'horgre': 124,
'horlar': 125, 'houfin': 126, 'houspa': 127, 'houwre': 128, 'indbun': 129,
'juntit1': 130, 'killde': 131, 'labwoo': 132, 'larspa': 133, 'lazbun': 134,
'leabit': 135, 'leafly': 136, 'leasan': 137, 'lecthr': 138, 'lesgol': 139,
'lesnig': 140, 'lesyel': 141, 'lewwoo': 142, 'linspa': 143, 'lobcur': 144,
'lobdow': 145, 'logshr': 146, 'lotduc': 147, 'louwat': 148, 'macwar': 149,
'magwar': 150, 'mallar3': 151, 'marwre': 152, 'merlin': 153, 'moublu': 154,
'mouchi': 155, 'moudov': 156, 'norcar': 157, 'norfli': 158, 'norhar2': 159,
'normoc': 160, 'norpar': 161, 'norpin': 162, 'norsho': 163, 'norwat': 164,
'nrwswa': 165, 'nutwoo': 166, 'olsfly': 167, 'orcwar': 168, 'osprey': 169,
'ovenbi1': 170, 'palwar': 171, 'pasfly': 172, 'pecsan': 173, 'perfal': 174,
'phaino': 175, 'pibgre': 176, 'pilwoo': 177, 'pingro': 178, 'pinjay': 179,
'pinsis': 180, 'pinwar': 181, 'plsvir': 182, 'prawar': 183, 'purfin': 184,
'pygnut': 185, 'rebmer': 186, 'rebnut': 187, 'rebsap': 188, 'rebwoo': 189,
'redcro': 190, 'redhea': 191, 'reevir1': 192, 'renpha': 193, 'reshaw': 194,
'rethaw': 195, 'rewbla': 196, 'ribgul': 197, 'rinduc': 198, 'robgro': 199,
'rocpig': 200, 'rocwre': 201, 'rthhum': 202, 'ruckin': 203, 'rudduc': 204,
'rufgro': 205, 'rufhum': 206, 'rusbla': 207, 'sagspa1': 208, 'sagthr': 209,
'savspa': 210, 'saypho': 211, 'scatan': 212, 'scoori': 213, 'semplo': 214,
'semsan': 215, 'sheowl': 216, 'shshaw': 217, 'snobun': 218, 'snogoo': 219,
'solsan': 220, 'sonspa': 221, 'sora': 222, 'sposan': 223, 'spotow': 224,
'stejay': 225, 'swahaw': 226, 'swaspa': 227, 'swathr': 228, 'treswa': 229,
'truswa': 230, 'tuftit': 231, 'tunswa': 232, 'veery': 233, 'vesspa': 234,
'vigswa': 235, 'warvir': 236, 'wesblu': 237, 'wesgre': 238, 'weskin': 239,
'wesmea': 240, 'wessan': 241, 'westan': 242, 'wewpew': 243, 'whbnut': 244,
'whcspa': 245, 'whfibi': 246, 'whtspa': 247, 'whtswi': 248, 'wilfly': 249,
'wilsni1': 250, 'wiltur': 251, 'winwre3': 252, 'wlswar': 253, 'wooduc': 254,
'wooscj2': 255, 'woothr': 256, 'y00475': 257, 'yebfly': 258, 'yebsap': 259,
'yehbla': 260, 'yelwar': 261, 'yerwar': 262, 'yetvir': 263
}
INV_BIRD_CODE = {v: k for k, v in BIRD_CODE.items() }
class SmoothBCEwLogits(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=0.0):
super().__init__(weight=weight, reduction=reduction)
 self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
@staticmethod
def _smooth(targets:torch.Tensor, n_labels:int, smoothing=0.0):
assert 0 <= smoothing < 1
with torch.no_grad() :
targets = targets *(1.0 - smoothing)+ 0.5 * smoothing
return targets
def forward(self, inputs, targets):
targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1),
self.smoothing)
 loss = F.binary_cross_entropy_with_logits(inputs, targets,self.weight)
 if self.reduction == 'sum':
loss = loss.sum()
elif self.reduction == 'mean':
loss = loss.mean()
return loss<categorify>
class SplAtConv2d(Module):

 def __init__(self, in_channels, channels, kernel_size, stride=(1, 1), padding=(0, 0),
dilation=(1, 1), groups=1, bias=True,
radix=2, reduction_factor=4,
rectify=False, rectify_avg=False, norm_layer=None,
dropblock_prob=0.0, **kwargs):
super(SplAtConv2d, self ).__init__()
padding = _pair(padding)
 self.rectify = rectify and(padding[0] > 0 or padding[1] > 0)
 self.rectify_avg = rectify_avg
inter_channels = max(in_channels*radix//reduction_factor, 32)
 self.radix = radix
self.cardinality = groups
self.channels = channels
self.dropblock_prob = dropblock_prob
if self.rectify:
self.conv = RFConv2d(in_channels, channels*radix, kernel_size, stride, padding, dilation,
groups=groups*radix, bias=bias, average_mode=rectify_avg, **kwargs)
 else:
self.conv = Conv2d(in_channels, channels*radix, kernel_size, stride, padding, dilation,
groups=groups*radix, bias=bias, **kwargs)
 self.use_bn = norm_layer is not None
if self.use_bn:
self.bn0 = norm_layer(channels*radix)
 self.relu = ReLU(inplace=True)
 self.fc1 = Conv2d(channels, inter_channels, 1, groups=self.cardinality)
 if self.use_bn:
self.bn1 = norm_layer(inter_channels)
 self.fc2 = Conv2d(inter_channels, channels*radix, 1, groups=self.cardinality)
 if dropblock_prob > 0.0:
self.dropblock = DropBlock2D(dropblock_prob, 3)
 self.rsoftmax = rSoftMax(radix, groups)
 def forward(self, x):
x = self.conv(x)
 if self.use_bn:
x = self.bn0(x)
 if self.dropblock_prob > 0.0:
x = self.dropblock(x)
 x = self.relu(x)
 batch, rchannel = x.shape[:2]
if self.radix > 1:
if torch.__version__ < '1.5':
splited = torch.split(x, int(rchannel//self.radix), dim=1)
 else:
splited = torch.split(x, rchannel//self.radix, dim=1)
 gap = sum(splited)
 else:
gap = x
gap = F.adaptive_avg_pool2d(gap, 1)
 gap = self.fc1(gap)
 if self.use_bn:
gap = self.bn1(gap)
 gap = self.relu(gap)
 atten = self.fc2(gap)
 atten = self.rsoftmax(atten ).view(batch, -1, 1, 1)
 if self.radix > 1:
if torch.__version__ < '1.5':
attens = torch.split(atten, int(rchannel//self.radix), dim=1)
 else:
attens = torch.split(atten, rchannel//self.radix, dim=1)
 out = sum([att*split for(att, split)in zip(attens, splited)])
 else:
out = atten * x
return out.contiguous()
class rSoftMax(nn.Module):
def __init__(self, radix, cardinality):
super().__init__()
self.radix = radix
self.cardinality = cardinality
def forward(self, x):
batch = x.size(0)
 if self.radix > 1:
x = x.view(batch, self.cardinality, self.radix, -1 ).transpose(1, 2)
 x = F.softmax(x, dim=1)
 x = x.reshape(batch, -1)
 else:
x = torch.sigmoid(x)
 return x
def multifold_indexer(train,target_columns,n_splits=10,random_state=12347,**kwargs):
folds = train.copy()
mskf = MultilabelStratifiedKFold(n_splits=n_splits,random_state=random_state,**kwargs)
 folds[ 'kfold']=0
for f,(t_idx, v_idx)in enumerate(mskf.split(X=train, y=train[target_columns])) :
folds.iloc[v_idx,-1] = int(f)
 folds['kfold'] = folds['kfold'].astype(int)
 return folds 
<choose_model_class>
class DropBlock2D(object):
def __init__(self, *args, **kwargs):
raise NotImplementedError
class GlobalAvgPool2d(nn.Module):
def __init__(self):

 super(GlobalAvgPool2d, self ).__init__()
def forward(self, inputs):
return nn.functional.adaptive_avg_pool2d(inputs, 1 ).view(inputs.size(0), -1)
 class Bottleneck(nn.Module):

 expansion = 4
def __init__(self, inplanes, planes, stride=1, downsample=None,
radix=1, cardinality=1, bottleneck_width=64,
avd=False, avd_first=False, dilation=1, is_first=False,
rectified_conv=False, rectify_avg=False,
norm_layer=None, dropblock_prob=0.0, last_gamma=False):
super(Bottleneck, self ).__init__()
group_width = int(planes *(bottleneck_width / 64.))* cardinality
self.conv1 = nn.Conv2d(inplanes, group_width, kernel_size=1, bias=False)
 self.bn1 = norm_layer(group_width)
 self.dropblock_prob = dropblock_prob
self.radix = radix
self.avd = avd and(stride > 1 or is_first)
 self.avd_first = avd_first
if self.avd:
self.avd_layer = nn.AvgPool2d(3, stride, padding=1)
 stride = 1
if dropblock_prob > 0.0:
self.dropblock1 = DropBlock2D(dropblock_prob, 3)
 if radix == 1:
self.dropblock2 = DropBlock2D(dropblock_prob, 3)
 self.dropblock3 = DropBlock2D(dropblock_prob, 3)
 if radix >= 1:
self.conv2 = SplAtConv2d(
 group_width, group_width, kernel_size=3,
stride=stride, padding=dilation,
dilation=dilation, groups=cardinality, bias=False,
radix=radix, rectify=rectified_conv,
rectify_avg=rectify_avg,
norm_layer=norm_layer,
dropblock_prob=dropblock_prob)
 elif rectified_conv:
self.conv2 = RFConv2d(
 group_width, group_width, kernel_size=3, stride=stride,
padding=dilation, dilation=dilation,
groups=cardinality, bias=False,
average_mode=rectify_avg)
 self.bn2 = norm_layer(group_width)
 else:
self.conv2 = nn.Conv2d(
 group_width, group_width, kernel_size=3, stride=stride,
padding=dilation, dilation=dilation,
groups=cardinality, bias=False)
 self.bn2 = norm_layer(group_width)
 self.conv3 = nn.Conv2d(
 group_width, planes * 4, kernel_size=1, bias=False)
 self.bn3 = norm_layer(planes*4)
 if last_gamma:
zeros_(self.bn3.weight)
 self.relu = nn.ReLU(inplace=True)
 self.downsample = downsample
self.dilation = dilation
self.stride = stride
def forward(self, x):
residual = x
out = self.conv1(x)
 out = self.bn1(out)
 if self.dropblock_prob > 0.0:
out = self.dropblock1(out)
 out = self.relu(out)
 if self.avd and self.avd_first:
out = self.avd_layer(out)
 out = self.conv2(out)
 if self.radix == 0:
out = self.bn2(out)
 if self.dropblock_prob > 0.0:
out = self.dropblock2(out)
 out = self.relu(out)
 if self.avd and not self.avd_first:
out = self.avd_layer(out)
 out = self.conv3(out)
 out = self.bn3(out)
 if self.dropblock_prob > 0.0:
out = self.dropblock3(out)
 if self.downsample is not None:
residual = self.downsample(x)
 out += residual
out = self.relu(out)
 return out
class ResNet(nn.Module):

 def __init__(self, block, layers, radix=1, groups=1, bottleneck_width=64,
num_classes=1000, dilated=False, dilation=1,
deep_stem=False, stem_width=64, avg_down=False,
rectified_conv=False, rectify_avg=False,
avd=False, avd_first=False,
final_drop=0.0, dropblock_prob=0,
last_gamma=False, norm_layer=nn.BatchNorm2d):
self.cardinality = groups
self.bottleneck_width = bottleneck_width
self.inplanes = stem_width*2 if deep_stem else 64
self.avg_down = avg_down
self.last_gamma = last_gamma
self.radix = radix
self.avd = avd
self.avd_first = avd_first
super(ResNet, self ).__init__()
self.rectified_conv = rectified_conv
self.rectify_avg = rectify_avg
if rectified_conv:
conv_layer = RFConv2d
else:
conv_layer = nn.Conv2d
conv_kwargs = {'average_mode': rectify_avg} if rectified_conv else {}
if deep_stem:
self.conv1 = nn.Sequential(
 conv_layer(3, stem_width, kernel_size=3, stride=2, padding=1, bias=False, **conv_kwargs),
norm_layer(stem_width),
nn.ReLU(inplace=True),
conv_layer(stem_width, stem_width, kernel_size=3, stride=1, padding=1, bias=False, **conv_kwargs),
norm_layer(stem_width),
nn.ReLU(inplace=True),
conv_layer(stem_width, stem_width*2, kernel_size=3, stride=1, padding=1, bias=False, **conv_kwargs), 
)
 else:
self.conv1 = conv_layer(3, 64, kernel_size=7, stride=2, padding=3,
bias=False, **conv_kwargs)
 self.bn1 = norm_layer(self.inplanes)
 self.relu = nn.ReLU(inplace=True)
 self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
 self.layer1 = self._make_layer(block, 64, layers[0], norm_layer=norm_layer, is_first=False)
 self.layer2 = self._make_layer(block, 128, layers[1], stride=2, norm_layer=norm_layer)
 if dilated or dilation == 4:
self.layer3 = self._make_layer(block, 256, layers[2], stride=1,
dilation=2, norm_layer=norm_layer,
dropblock_prob=dropblock_prob)
 self.layer4 = self._make_layer(block, 512, layers[3], stride=1,
dilation=4, norm_layer=norm_layer,
dropblock_prob=dropblock_prob)
 elif dilation==2:
self.layer3 = self._make_layer(block, 256, layers[2], stride=2,
dilation=1, norm_layer=norm_layer,
dropblock_prob=dropblock_prob)
 self.layer4 = self._make_layer(block, 512, layers[3], stride=1,
dilation=2, norm_layer=norm_layer,
dropblock_prob=dropblock_prob)
 else:
self.layer3 = self._make_layer(block, 256, layers[2], stride=2,
norm_layer=norm_layer,
dropblock_prob=dropblock_prob)
 self.layer4 = self._make_layer(block, 512, layers[3], stride=2,
norm_layer=norm_layer,
dropblock_prob=dropblock_prob)
 self.avgpool = GlobalAvgPool2d()
self.drop = nn.Dropout(final_drop)if final_drop > 0.0 else None
self.fc = nn.Linear(512 * block.expansion, num_classes)
 for m in self.modules() :
if isinstance(m, nn.Conv2d):
n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
m.weight.data.normal_(0, math.sqrt(2./ n))
elif isinstance(m, norm_layer):
m.weight.data.fill_(1)
 m.bias.data.zero_()
def _make_layer(self, block, planes, blocks, stride=1, dilation=1, norm_layer=None,
dropblock_prob=0.0, is_first=True):
downsample = None
if stride != 1 or self.inplanes != planes * block.expansion:
down_layers = []
if self.avg_down:
if dilation == 1:
down_layers.append(nn.AvgPool2d(kernel_size=stride, stride=stride,
ceil_mode=True, count_include_pad=False))
else:
down_layers.append(nn.AvgPool2d(kernel_size=1, stride=1,
ceil_mode=True, count_include_pad=False))
down_layers.append(nn.Conv2d(self.inplanes, planes * block.expansion,
kernel_size=1, stride=1, bias=False))
else:
down_layers.append(nn.Conv2d(self.inplanes, planes * block.expansion,
kernel_size=1, stride=stride, bias=False))
down_layers.append(norm_layer(planes * block.expansion))
downsample = nn.Sequential(*down_layers)
 layers = []
if dilation == 1 or dilation == 2:
layers.append(block(self.inplanes, planes, stride, downsample=downsample,
radix=self.radix, cardinality=self.cardinality,
bottleneck_width=self.bottleneck_width,
avd=self.avd, avd_first=self.avd_first,
dilation=1, is_first=is_first, rectified_conv=self.rectified_conv,
rectify_avg=self.rectify_avg,
norm_layer=norm_layer, dropblock_prob=dropblock_prob,
last_gamma=self.last_gamma))
elif dilation == 4:
layers.append(block(self.inplanes, planes, stride, downsample=downsample,
radix=self.radix, cardinality=self.cardinality,
bottleneck_width=self.bottleneck_width,
avd=self.avd, avd_first=self.avd_first,
dilation=2, is_first=is_first, rectified_conv=self.rectified_conv,
rectify_avg=self.rectify_avg,
norm_layer=norm_layer, dropblock_prob=dropblock_prob,
last_gamma=self.last_gamma))
else:
raise RuntimeError("=> unknown dilation size: {}".format(dilation))
self.inplanes = planes * block.expansion
for i in range(1, blocks):
layers.append(block(self.inplanes, planes,
radix=self.radix, cardinality=self.cardinality,
bottleneck_width=self.bottleneck_width,
avd=self.avd, avd_first=self.avd_first,
dilation=dilation, rectified_conv=self.rectified_conv,
rectify_avg=self.rectify_avg,
norm_layer=norm_layer, dropblock_prob=dropblock_prob,
last_gamma=self.last_gamma))
return nn.Sequential(*layers)
 def forward(self, x):
x = self.conv1(x)
 x = self.bn1(x)
 x = self.relu(x)
 x = self.maxpool(x)
 x = self.layer1(x)
 x = self.layer2(x)
 x = self.layer3(x)
 x = self.layer4(x)
 x = self.avgpool(x)
 x = torch.flatten(x, 1)
 if self.drop:
x = self.drop(x)
 x = self.fc(x)
 return x
class Model(nn.Module):
def __init__(self, num_features, num_targets, hidden_size1=388,hidden_size2=512,drop_rate1=0.0,drop_rate2=0.3,drop_rate3=0.3):
super(Model, self ).__init__()
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dropout1 = nn.Dropout(drop_rate1)
 self.dense1 = nn.utils.weight_norm(nn.Linear(num_features, hidden_size1))
self.batch_norm2 = nn.BatchNorm1d(hidden_size1)
 self.dropout2 = nn.Dropout(drop_rate2)
 self.dense2 = nn.utils.weight_norm(nn.Linear(hidden_size1, hidden_size2))
self.batch_norm3 = nn.BatchNorm1d(hidden_size2)
 self.dropout3 = nn.Dropout(drop_rate3)
 self.dense3 = nn.utils.weight_norm(nn.Linear(hidden_size2, num_targets))
def forward(self, x):
x = self.batch_norm1(x)
 x = self.dropout1(x)
 x = F.relu(self.dense1(x))
x = self.batch_norm2(x)
 x = self.dropout2(x)
 x = F.relu(self.dense2(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = self.dense3(x)
 return x<train_model>
def get_model(args: tp.Dict):
model = ResNet(
 Bottleneck, [3, 4, 6, 3],
radix=1, groups=1, bottleneck_width=64,
deep_stem=True, stem_width=32, avg_down=True,
avd=True, avd_first=True)
 del model.fc
model.fc = nn.Sequential(
 nn.Linear(2048, 1024), nn.ReLU() , nn.Dropout(p=0.2),
nn.Linear(1024, 1024), nn.ReLU() , nn.Dropout(p=0.2),
nn.Linear(1024, args["num_classes"]))
state_dict = torch.load(args["trained_weights"])
 model.load_state_dict(state_dict)
 device = torch.device("cuda")
 model.to(device)
 model.eval()
return model
def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):
model.train()
final_loss = 0
for data in dataloader:
optimizer.zero_grad()
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
if not scheduler.__class__ == torch.optim.lr_scheduler.ReduceLROnPlateau:
scheduler.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 return final_loss
def valid_fn(model, scheduler, loss_fn, dataloader, device):
model.eval()
final_loss = 0
valid_preds = []
for data in dataloader:
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 if scheduler.__class__ == torch.optim.lr_scheduler.ReduceLROnPlateau:
scheduler.step(final_loss)
 return final_loss, valid_preds
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
inputs = data['x'].to(device)
 with torch.no_grad() :
outputs = model(inputs)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds 
<define_variables>
def prediction_for_clip(test_df: pd.DataFrame,
clip: np.ndarray,
model: ResNet,
mel_params: dict,
threshold=0.5):
dataset = TestDataset(df=test_df,
clip=clip,
img_size=224,
melspectrogram_parameters=mel_params)
 loader = data.DataLoader(dataset, batch_size=1, shuffle=False)
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 model.eval()
prediction_dict = {}
for image, row_id, site in progress_bar(loader):
site = site[0]
row_id = row_id[0]
if site in {"site_1", "site_2"}:
image = image.to(device)
 with torch.no_grad() :
prediction = F.sigmoid(model(image))
proba = prediction.detach().cpu().numpy().reshape(-1)
 events = proba >= threshold
labels = np.argwhere(events ).reshape(-1 ).tolist()
if len(labels)== 0:
prediction_dict[row_id] = "nocall"
else:
labels_str_list = list(map(lambda x: INV_BIRD_CODE[x], labels))
label_string = " ".join(labels_str_list)
 prediction_dict[row_id] = label_string
if site in {"site_3"}:
image = image.squeeze(0)
 print(image.shape)
 batch_size = 16
whole_size = image.size(0)
 if whole_size % batch_size == 0:
n_iter = whole_size // batch_size
else:
n_iter = whole_size // batch_size + 1
all_events = set()
for batch_i in range(n_iter):
batch = image[batch_i * batch_size:(batch_i + 1)* batch_size]
if batch.ndim == 3:
batch = batch.unsqueeze(0)
 batch = batch.to(device)
 with torch.no_grad() :
prediction = F.sigmoid(model(batch))
proba = prediction.detach().cpu().numpy()
events = proba >= threshold
for i in range(len(events)) :
event = events[i, :]
labels = np.argwhere(event ).reshape(-1 ).tolist()
for label in labels:
all_events.add(label)
 labels = list(all_events)
 if len(labels)== 0:
prediction_dict[row_id] = "nocall"
else:
labels_str_list = list(map(lambda x: INV_BIRD_CODE[x], labels))
label_string = " ".join(labels_str_list)
 prediction_dict[row_id] = label_string
if site in {"example"}:
nsec = 1
image = image.squeeze(0)
 print(image.shape)
 batch_size = 16
whole_size = image.size(0)
 if whole_size % batch_size == 0:
n_iter = whole_size // batch_size
else:
n_iter = whole_size // batch_size + 1
for batch_i in range(n_iter):
batch = image[batch_i * batch_size:(batch_i + 1)* batch_size]
if batch.ndim == 3:
batch = batch.unsqueeze(0)
 batch = batch.to(device)
 with torch.no_grad() :
prediction = F.sigmoid(model(batch))
proba = prediction.detach().cpu().numpy()
events =(proba >= threshold)
 for i in range(len(events)) :
row_id_sec = row_id+'_'+str(int(nsec*5))
nsec += 1
event = events[i, :]
all_events = set()
labels = np.argwhere(event ).reshape(-1 ).tolist()
for label in labels:
all_events.add(label)
 labels = list(all_events)
 if len(labels)== 0:
prediction_dict[row_id_sec] = "nocall"
else:
labels_str_list = list(map(lambda x: INV_BIRD_CODE[x], labels))
label_string = " ".join(labels_str_list)
 prediction_dict[row_id_sec] = label_string
return prediction_dict
exp_name = 'DAE'<define_variables>
def prediction(test_df: pd.DataFrame,
test_audio: Path,
model_config: dict,
mel_params: dict,
target_sr: int,
threshold=0.5):
model = get_model(model_config)
 unique_audio_id = test_df.audio_id.unique()
warnings.filterwarnings("ignore")
 prediction_dfs = []
for audio_id in unique_audio_id:
with timer(f"Loading {audio_id}"):
clip, _ = librosa.load(test_audio /(audio_id + ".mp3"),
sr=target_sr,
mono=True,
res_type="kaiser_fast")
 test_df_for_audio_id = test_df.query(
 f"audio_id == '{audio_id}'" ).reset_index(drop=True)
 with timer(f"Prediction on {audio_id}"):
prediction_dict = prediction_for_clip(test_df_for_audio_id,
clip=clip,
model=model,
mel_params=mel_params,
threshold=threshold)
 row_id = list(prediction_dict.keys())
 birds = list(prediction_dict.values())
 prediction_df = pd.DataFrame({
"row_id": row_id,
"birds": birds
})
 prediction_dfs.append(prediction_df)
 prediction_df = pd.concat(prediction_dfs, axis=0, sort=False ).reset_index(drop=True)
 return prediction_df
VALIDATE = True<train_model>
test_sample = pd.read_csv(".. /input/birdsong-recognition/example_test_audio_summary.csv")
 test_sample.head()
test2 = test_sample.copy()
test2['audio_id'] = test2['filename'].copy()
test2['site'] = 'example'
test2['row_id'] = test2['site'] + '_' + test2['audio_id'] + '_' + test2['seconds'].astype(str)
 test2['seconds'] = test2['seconds'].astype(np.float32)
 test2 = test2[['site','row_id','seconds','audio_id','birds']]
test2.head()
def run_training(X_train,y_train,X_valid,y_valid,X_test,fold, seed,verbose=False,ct=None,**kwargs):
seed_everything(seed)
 train_dataset = MoADataset(X_train, y_train)
 valid_dataset = MoADataset(X_valid, y_valid)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features= X_train.shape[1] ,
num_targets= y_train.shape[1],hidden_size1=hidden_size1,hidden_size2=hidden_size2,
**kwargs 
)
 model.to(DEVICE)
 optimizer = torch.optim.Adam(model.parameters() , lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY)
 scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer,patience=3)
 loss_val = nn.BCEWithLogitsLoss()
loss_tr = SmoothBCEwLogits(smoothing =0.001)
 early_stopping_steps = EARLY_STOPPING_STEPS
early_step = 0
best_loss = np.inf
if 0:
train_loss = train_fn(model, optimizer,scheduler, loss_tr, trainloader, DEVICE)
 if verbose &(epoch%10==0):
print(f"FOLD: {fold}, EPOCH: {epoch}, train_loss: {train_loss}")
 valid_loss, valid_preds = valid_fn(model,scheduler, loss_val, validloader, DEVICE)
 if verbose &(epoch%10==0):
print(f"FOLD: {fold}, EPOCH: {epoch}, valid_loss: {valid_loss}")
 if valid_loss < best_loss:
best_loss = valid_loss
oof = valid_preds
elif(EARLY_STOP == True):
early_step += 1
if(early_step >= early_stopping_steps):
break
testdataset = TestDataset(X_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features= X_train.shape[1] ,
num_targets= y_train.shape[1],
hidden_size1=hidden_size1,
hidden_size2=hidden_size2,
**kwargs 
)
 model.load_state_dict(torch.load(f"{BOOK2}{exp_name}_SEED{seed}_FOLD{fold}_CT{ct}.pth"))
model.to(DEVICE)
 print('LOAD MODEL_FOLD%i_CT%i'%(fold,ct))
if VALIDATE:
valid_loss, oof = valid_fn(model,scheduler, loss_val, validloader, DEVICE)
 else:
oof = np.zeros(( len(X_valid),y_train.shape[1]))
predictions = inference_fn(model, testloader, DEVICE)
 return oof, predictions 
<init_hyperparams>
test_tosub = test2[['site','audio_id']].drop_duplicates()
test_tosub['row_id'] = test_tosub['site'] + '_' + test_tosub['audio_id']
test_tosub['seconds'] = np.nan
test_tosub = test_tosub[['site','row_id', 'seconds', 'audio_id']]
test_tosub
DEVICE = torch.device('cuda:0')
 EPOCHS = 36
BATCH_SIZE = 1024
LEARNING_RATE = 6e-3
WEIGHT_DECAY = 1e-5
NFOLDS = 7
EARLY_STOPPING_STEPS = 10
EARLY_STOP = False
hidden_size1=2048
hidden_size2=2048<define_variables>
test_tosub.iloc[0,3] = 'BLKFR-10-CPL_20190611_093000.pt540'
test_tosub.iloc[1,3] = 'ORANGE-7-CAP_20190606_093000.pt623'
test_tosub
def run_k_fold(folds,target_cols,test,transformers_list,NFOLDS, seed,verbose=False,ct=None, **kwargs):
train = folds
test_ = test
oof = train[target_cols].copy()
oof = oof*0
predictions = pd.DataFrame(0,columns=target_cols,index=test.index)
 for fold in range(NFOLDS):
train_df = train[train['kfold'] != fold]
valid_df = train[train['kfold'] == fold]
feature_cols = [col for col in train_df.columns if not(col in target_cols+['kfold'])]
pipeline_val = make_pipeline(*transformers_list)
 X_train, y_train = train_df[feature_cols], train_df[target_cols]
X_valid, y_valid = valid_df[feature_cols], valid_df[target_cols].values
X_train = pipeline_val.fit_transform(X_train,y_train)
 feature_cols = [col for col in X_train.columns if not(col in target_cols+['kfold'])]
X_train = X_train.values
X_valid = pipeline_val.transform(X_valid)
 valid_index = X_valid.index
X_valid = X_valid.values
y_train = y_train.values
X_test = pipeline_val.transform(test_)
 test_index = X_test.index
X_test = X_test[feature_cols].values
oof_, pred_ = run_training(X_train,y_train,X_valid,y_valid,X_test,fold, seed,verbose,ct=ct,**kwargs)
 oof.loc[valid_index] = oof_
predictions.loc[test_index] += pred_ / NFOLDS
return oof, predictions<find_best_params>
test3 = test2.copy()
BIRD_CODE['nocall'] = 264
res = []
for thr in np.linspace(0.10,1.00,19):
test2 = test3.copy()
TEST_AUDIO_DIR2 = Path('.. /input/birdsong-recognition/example_test_audio/')
 submission_test = prediction(test_df=test_tosub,
test_audio=TEST_AUDIO_DIR2,
model_config=model_config,
mel_params=melspectrogram_parameters,
target_sr=TARGET_SR,
threshold=thr)
 submission_test.rename(columns={'birds': 'birds_pred'}, inplace=True)
 test2 = test2.merge(submission_test,how="left",on="row_id")
 test2['birds'] = test2['birds'].fillna("nocall")
 labels_hot = np.zeros(( len(test2), 265), dtype=int)
 preds_hot = np.zeros(( len(test2), 265), dtype=int)
 for nrow in range(len(labels_hot)) :
row = test2.iloc[nrow]
for label in row['birds'].split(' '):
if label in BIRD_CODE.keys() :
labels_hot[nrow,BIRD_CODE[label]] = 1
if labels_hot[nrow,:].sum() <1.0:
labels_hot[nrow,BIRD_CODE["nocall"]] = 1
for label in row['birds_pred'].split(' '):
preds_hot[nrow,BIRD_CODE[label]] = 1
f1score = f1_score(labels_hot,preds_hot,average='micro')
 res.append([thr, f1score])
params={}<load_from_csv>
res = pd.DataFrame(res,columns=['threshold','f1-score'])
 res
k = 0
SEED = [0]
Dae0_2 = DaeAdder(filename='features_0.2_altogether_fold%i.fth'%k)
 transformers_list=[map_dose,map_time,Dae0_2,SuppressControls() ]
train,target_cols = train_short_form_loader('.. /input/lish-moa/train_features.csv','.. /input/lish-moa/train_targets_scored.csv')
 test = test_short_form_loader(".. /input/lish-moa/test_features.csv")
 pipeline_test = make_pipeline(*transformers_list)
 pipeline_test,train , test = apply_pipe_together(pipeline_test,train,test,ct=k)
 oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for k in range(CTT):
if k!=0:
Dae0_2 = DaeAdder(filename='features_0.2_altogether_fold%i.fth'%k)
 transformers_list=[map_dose,map_time,Dae0_2,SuppressControls() ]
train,target_cols = train_short_form_loader('.. /input/lish-moa/train_features.csv','.. /input/lish-moa/train_targets_scored.csv')
 test = test_short_form_loader(".. /input/lish-moa/test_features.csv")
 pipeline_test = make_pipeline(*transformers_list)
 pipeline_test,train , test = apply_pipe_together(pipeline_test,train,test,ct=k)
 transformers_list=[IdentityTransformer() ]
for seed in SEED:
folds = multifold_indexer(train,target_cols,n_splits=NFOLDS)
 oof_, predictions_ = run_k_fold(folds,target_cols,test,transformers_list,NFOLDS, seed,verbose=True,ct=k,**params)
 oof += oof_ / len(SEED)/ CTT
predictions += predictions_ / len(SEED)/ CTT
print()
test[target_cols] = predictions<compute_test_metric>
threshold = res['threshold'].values[res['f1-score'].argmax() ]
threshold
<feature_engineering><EOS> 
submission = prediction(test_df=test,
test_audio=TEST_AUDIO_DIR,
model_config=model_config,
mel_params=melspectrogram_parameters,
target_sr=TARGET_SR,
threshold=threshold)
 submission.to_csv("submission.csv", index=False )