Kaggle competition: Cornell Birdcall Identification
<SOS> metric: MeanFScoreBeta Kaggle data source: cornell-birdcall-identification<categorify>
!pip uninstall torch --y
!pip uninstall torchaudio --y
<train_model>
!pip install.. /input/pytorch-160-with-torchvision-070/torch-1.6.0cu101-cp37-cp37m-linux_x86_64.whl
!pip install.. /input/torchaudio/torchaudio-0.6.0-cp37-cp37m-manylinux1_x86_64.whl
!pip install.. /input/bird-panns/torchlibrosa-master/torchlibrosa-master/ > /dev/null
pd.options.display.max_rows = 500
pd.options.display.max_columns = 500
class MultilabelStratifiedGroupKFold(_BaseKFold):
def __init__(self, n_splits=5, random_state=None, shuffle=False):
super().__init__(n_splits=n_splits, random_state=random_state, shuffle=shuffle)
 def _iter_test_indices(self, X=None, y=None, groups=None):
cv = MultilabelStratifiedKFold(
 n_splits=self.n_splits,
random_state=self.random_state,
shuffle=self.shuffle, 
)
 value_counts = groups.value_counts()
regluar_indices = value_counts.loc[ 
(value_counts == 6)|(value_counts == 12)|(value_counts == 18)
 ].index.sort_values()
irregluar_indices = value_counts.loc[ 
(value_counts != 6)&(value_counts != 12)&(value_counts != 18)
 ].index.sort_values()
group_to_fold = {}
tmp = y.groupby(groups ).mean().loc[regluar_indices]
for fold,(_, test)in enumerate(cv.split(tmp, tmp)) :
group_to_fold.update({group: fold for group in tmp.index[test]})
 sample_to_fold = {}
tmp = y.loc[groups.isin(irregluar_indices)]
for fold,(_, test)in enumerate(cv.split(tmp, tmp)) :
sample_to_fold.update({sample: fold for sample in tmp.index[test]})
 folds = groups.map(group_to_fold)
 is_na = folds.isna()
folds[is_na] = folds[is_na].index.map(sample_to_fold ).values
for i in range(self.n_splits):
yield np.where(folds == i)[0]<load_from_csv>
args = {
'n_mels': 224,
'hop_length': 384,
'std_sr': 32000,
'threshold':.6
}
root = os.path.join('.. /input', "birdsong-recognition")
 test_dir = os.path.join(root, 'test_audio')
 print(test_dir)
 if os.path.isdir(test_dir):
print('Running actual submission!')
 test_df = pd.read_csv(os.path.join(root, 'test.csv'))
sub = pd.read_csv(".. /input/birdsong-recognition/sample_submission.csv")
 else:
print('Running debug submission')
 test_dir = os.path.join('.. /input', 'birdcall-check', 'test_audio')
 test_df = pd.read_csv(os.path.join('.. /input', 'birdcall-check', 'test.csv'))
sub = pd.read_csv('.. /input/inference-pytorch-birdcall-resnet-baseline/submission.csv')
 print('Naturally no test dir found')
 sub.to_csv("submission.csv", index=False )
dtype = {"cp_type": "category", "cp_dose": "category"}
index_col = "sig_id"
train_features = pd.read_csv(
 ".. /input/lish-moa/train_features.csv", dtype=dtype, index_col=index_col 
)
 X = train_features.select_dtypes("number")
 Y = pd.read_csv(".. /input/lish-moa/train_targets_scored.csv", index_col=index_col)
 groups = pd.read_csv(
 ".. /input/lish-moa/train_drug.csv", index_col=index_col, squeeze=True 
)
 columns = Y.columns<features_selection>
BIRD_CODE = {
'aldfly': 0, 'ameavo': 1, 'amebit': 2, 'amecro': 3, 'amegfi': 4,
'amekes': 5, 'amepip': 6, 'amered': 7, 'amerob': 8, 'amewig': 9,
'amewoo': 10, 'amtspa': 11, 'annhum': 12, 'astfly': 13, 'baisan': 14,
'baleag': 15, 'balori': 16, 'banswa': 17, 'barswa': 18, 'bawwar': 19,
'belkin1': 20, 'belspa2': 21, 'bewwre': 22, 'bkbcuc': 23, 'bkbmag1': 24,
'bkbwar': 25, 'bkcchi': 26, 'bkchum': 27, 'bkhgro': 28, 'bkpwar': 29,
'bktspa': 30, 'blkpho': 31, 'blugrb1': 32, 'blujay': 33, 'bnhcow': 34,
'boboli': 35, 'bongul': 36, 'brdowl': 37, 'brebla': 38, 'brespa': 39,
'brncre': 40, 'brnthr': 41, 'brthum': 42, 'brwhaw': 43, 'btbwar': 44,
'btnwar': 45, 'btywar': 46, 'buffle': 47, 'buggna': 48, 'buhvir': 49,
'bulori': 50, 'bushti': 51, 'buwtea': 52, 'buwwar': 53, 'cacwre': 54,
'calgul': 55, 'calqua': 56, 'camwar': 57, 'cangoo': 58, 'canwar': 59,
'canwre': 60, 'carwre': 61, 'casfin': 62, 'caster1': 63, 'casvir': 64,
'cedwax': 65, 'chispa': 66, 'chiswi': 67, 'chswar': 68, 'chukar': 69,
'clanut': 70, 'cliswa': 71, 'comgol': 72, 'comgra': 73, 'comloo': 74,
'commer': 75, 'comnig': 76, 'comrav': 77, 'comred': 78, 'comter': 79,
'comyel': 80, 'coohaw': 81, 'coshum': 82, 'cowscj1': 83, 'daejun': 84,
'doccor': 85, 'dowwoo': 86, 'dusfly': 87, 'eargre': 88, 'easblu': 89,
'easkin': 90, 'easmea': 91, 'easpho': 92, 'eastow': 93, 'eawpew': 94,
'eucdov': 95, 'eursta': 96, 'evegro': 97, 'fiespa': 98, 'fiscro': 99,
'foxspa': 100, 'gadwal': 101, 'gcrfin': 102, 'gnttow': 103, 'gnwtea': 104,
'gockin': 105, 'gocspa': 106, 'goleag': 107, 'grbher3': 108, 'grcfly': 109,
'greegr': 110, 'greroa': 111, 'greyel': 112, 'grhowl': 113, 'grnher': 114,
'grtgra': 115, 'grycat': 116, 'gryfly': 117, 'haiwoo': 118, 'hamfly': 119,
'hergul': 120, 'herthr': 121, 'hoomer': 122, 'hoowar': 123, 'horgre': 124,
'horlar': 125, 'houfin': 126, 'houspa': 127, 'houwre': 128, 'indbun': 129,
'juntit1': 130, 'killde': 131, 'labwoo': 132, 'larspa': 133, 'lazbun': 134,
'leabit': 135, 'leafly': 136, 'leasan': 137, 'lecthr': 138, 'lesgol': 139,
'lesnig': 140, 'lesyel': 141, 'lewwoo': 142, 'linspa': 143, 'lobcur': 144,
'lobdow': 145, 'logshr': 146, 'lotduc': 147, 'louwat': 148, 'macwar': 149,
'magwar': 150, 'mallar3': 151, 'marwre': 152, 'merlin': 153, 'moublu': 154,
'mouchi': 155, 'moudov': 156, 'norcar': 157, 'norfli': 158, 'norhar2': 159,
'normoc': 160, 'norpar': 161, 'norpin': 162, 'norsho': 163, 'norwat': 164,
'nrwswa': 165, 'nutwoo': 166, 'olsfly': 167, 'orcwar': 168, 'osprey': 169,
'ovenbi1': 170, 'palwar': 171, 'pasfly': 172, 'pecsan': 173, 'perfal': 174,
'phaino': 175, 'pibgre': 176, 'pilwoo': 177, 'pingro': 178, 'pinjay': 179,
'pinsis': 180, 'pinwar': 181, 'plsvir': 182, 'prawar': 183, 'purfin': 184,
'pygnut': 185, 'rebmer': 186, 'rebnut': 187, 'rebsap': 188, 'rebwoo': 189,
'redcro': 190, 'redhea': 191, 'reevir1': 192, 'renpha': 193, 'reshaw': 194,
'rethaw': 195, 'rewbla': 196, 'ribgul': 197, 'rinduc': 198, 'robgro': 199,
'rocpig': 200, 'rocwre': 201, 'rthhum': 202, 'ruckin': 203, 'rudduc': 204,
'rufgro': 205, 'rufhum': 206, 'rusbla': 207, 'sagspa1': 208, 'sagthr': 209,
'savspa': 210, 'saypho': 211, 'scatan': 212, 'scoori': 213, 'semplo': 214,
'semsan': 215, 'sheowl': 216, 'shshaw': 217, 'snobun': 218, 'snogoo': 219,
'solsan': 220, 'sonspa': 221, 'sora': 222, 'sposan': 223, 'spotow': 224,
'stejay': 225, 'swahaw': 226, 'swaspa': 227, 'swathr': 228, 'treswa': 229,
'truswa': 230, 'tuftit': 231, 'tunswa': 232, 'veery': 233, 'vesspa': 234,
'vigswa': 235, 'warvir': 236, 'wesblu': 237, 'wesgre': 238, 'weskin': 239,
'wesmea': 240, 'wessan': 241, 'westan': 242, 'wewpew': 243, 'whbnut': 244,
'whcspa': 245, 'whfibi': 246, 'whtspa': 247, 'whtswi': 248, 'wilfly': 249,
'wilsni1': 250, 'wiltur': 251, 'winwre3': 252, 'wlswar': 253, 'wooduc': 254,
'wooscj2': 255, 'woothr': 256, 'y00475': 257, 'yebfly': 258, 'yebsap': 259,
'yehbla': 260, 'yelwar': 261, 'yerwar': 262, 'yetvir': 263
}
INV_BIRD_CODE = {v: k for k, v in BIRD_CODE.items() }
c_prefix = "c-"
g_prefix = "g-"
c_columns = X.columns.str.startswith(c_prefix)
 g_columns = X.columns.str.startswith(g_prefix)
 X_stats_c = compute_row_statistics(X.loc[:, c_columns], prefix=c_prefix)
 X_stats_g = compute_row_statistics(X.loc[:, g_columns], prefix=g_prefix)
 with open(".. /input/preprocessor-fit/clipped_features.pkl", "rb")as f:
clipped_features = pickle.load(f)
 X = clipped_features.transform(X)
 X = pd.concat([X, X_stats_c, X_stats_g], axis=1)
<init_hyperparams>
fmin, fmax = 25, 11000
f1 = MelSpectrogram(sample_rate=args['std_sr'], n_fft=1024, hop_length=args['hop_length'], f_min=fmin, f_max=fmax, n_mels=args['n_mels'] ).cuda()
f2 = MelSpectrogram(sample_rate=args['std_sr'], n_fft=2048, hop_length=args['hop_length'], f_min=fmin, f_max=fmax, n_mels=args['n_mels'] ).cuda()
f3 = MelSpectrogram(sample_rate=args['std_sr'], n_fft=4096, hop_length=args['hop_length'], f_min=fmin, f_max=fmax, n_mels=args['n_mels'] ).cuda()
alpha = 4.0
batch_size = 32
factor = 0.5
n_seeds = 5
n_splits = 5
patience = 30
shuffle = True
params = {
"activation": "elu",
"kernel_initializer": "he_normal",
"label_smoothing": 5e-04,
"lr": 0.03,
"n_layers": 6,
"n_units": 256,
"rate": 0.3,
}
fit_params = {"epochs": 1_000, "verbose": 0}<create_dataframe>
class TestDataset(data.Dataset):
def __init__(self, df, spectrogram):
self.df = df
self.spectrogram = spectrogram
def __len__(self):
return len(self.df)
 def normalize(self, spect):
spect = spect.contiguous()
miu = spect.view(spect.size(0), -1 ).mean(-1 ).unsqueeze(-1 ).unsqueeze(-1)
 std = spect.view(spect.size(0), -1 ).std(-1 ).unsqueeze(-1 ).unsqueeze(-1)+ 1e-8
return(spect - miu)/ std / 2
def __getitem__(self, idx):
sample = self.df.loc[idx, :]
site = sample.site
row_id = sample.row_id
if site == "site_3":
y = self.spectrogram
len_y = y.shape[-1]
start = 0
end = args['std_sr'] * 5 // args['hop_length']
y_all = []
while len_y > start:
y_batch = self.normalize(y[:, :, start:end])
 if end >= len_y:
clip_length = args['std_sr'] * 5 // args['hop_length']
y_batch = self.normalize(y[:, :, -clip_length:])
 y_all.append(y_batch)
 break
start = end
end = end + args['std_sr'] * 5 // args['hop_length']
y_all.append(y_batch)
 y_all = torch.stack(y_all)
 return y_all, row_id, site
else:
end_seconds = int(sample.seconds)
 start_seconds = int(end_seconds - 5)
 start_index = args['std_sr'] * start_seconds // args['hop_length']
end_index = args['std_sr'] * end_seconds // args['hop_length']
y = self.spectrogram[:, :, start_index:end_index]
y = self.normalize(y)
 return y, row_id, site
def prediction_for_clip(dataset, models, threshold=0.5):
loader = data.DataLoader(dataset, batch_size=1, shuffle=False)
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 prediction_dict = {}
for image, row_id, site in loader:
site = site[0]
row_id = row_id[0]
image = image.to(device ).float()
preds = []
if site in {"site_1", "site_2"}:
image = image.to(device ).float()
with torch.no_grad() :
for model in models:
model.eval()
prediction = model(image)
 proba = prediction.view(-1 ).sigmoid().cpu().numpy()
preds.append(proba)
 else:
image = image.squeeze(0)
 batch_size = 16
whole_size = image.size(0)
 if whole_size % batch_size == 0:
n_iter = whole_size // batch_size
else:
n_iter = whole_size // batch_size + 1
probas = [[] for _ in range(len(models)) ]
all_events = set()
for batch_i in range(n_iter):
batch = image[batch_i * batch_size:(batch_i + 1)* batch_size]
if batch.ndim == 3:
batch = batch.unsqueeze(0)
 batch = batch.to(device)
 with torch.no_grad() :
for idx, model in enumerate(models):
model.eval()
prediction = model(batch)
 proba = prediction.view(batch.size(0), -1 ).sigmoid().cpu().numpy()
probas[idx].append(proba)
 for sub_probas in probas:
sub_probas = np.vstack(sub_probas)
 proba = sub_probas.max(0)
 preds.append(proba)
 prediction_dict[row_id] = preds
return prediction_dict
<define_variables>
model_paths = [
'.. /input/birdweights/c2__17.pt',
'.. /input/birdweights/c3_10_42.pt',
'.. /input/birdweights/c3_3_29.pt',
'.. /input/birdweights/c3_4_17.pt'
]
%%time
bias_initializer = -Y.mean(axis=0 ).apply(np.log ).values
bias_initializer = tf.keras.initializers.Constant(bias_initializer)
 Y_pred = np.zeros(( train_size, n_classes))
Y_pred = pd.DataFrame(Y_pred, columns=Y.columns, index=Y.index)
 for i in range(n_seeds):
set_seed(seed=i)
 cv = MultilabelStratifiedGroupKFold(
 n_splits=n_splits, random_state=i, shuffle=shuffle 
)
 for j,(train, valid)in enumerate(cv.split(X, Y[columns], groups)) :
model_path = f"model_seed_{i}_fold_{j}.h5"
model = build_mlp_classifier(
 n_features,
n_classes,
bias_initializer=bias_initializer,
**params, 
)
 generator = Cutmix(
 X.iloc[train], Y.iloc[train], alpha=alpha, batch_size=batch_size 
)
 callbacks = build_callbacks(model_path, factor=factor, patience=patience)
 history = model.fit(
 generator,
callbacks=callbacks,
validation_data=(X.iloc[valid], Y.iloc[valid]),
**fit_params, 
)
 model.load_weights(model_path)
 Y_pred.iloc[valid] += model.predict(X.iloc[valid])/ n_seeds
Y_pred[train_features["cp_type"] == "ctl_vehicle"] = 0.0
with open("Y_pred.pkl", "wb")as f:
pickle.dump(Y_pred[columns], f )<compute_test_metric>
models = []
for path in model_paths:
model = torch.jit.load(path, map_location='cpu' ).cuda()
models.append(model )
score(Y[columns], Y_pred[columns] )<import_modules>
prediction_dict = {}
for key, preds in submission.items() :
pred3 = preds[0]
pred4 = preds[1]
pred5 = preds[2]
pred6 = preds[3]
maxbird=3
threshold=0.35
if 'site_3' in key:
maxbird=8
threshold=0.45
en_pred = 0.25*pred3 +0.25*pred4 +0.25*pred5 +0.25*pred6
events =(en_pred>=threshold*1)
 labels = np.argsort(-en_pred)[:events.sum() ].tolist()
if len(labels)== 0:
prediction_dict[key] = "nocall"
else:
labels_str_list = list(map(lambda x: INV_BIRD_CODE[x], labels))
label_string = " ".join(sorted(labels_str_list[:maxbird]))
prediction_dict[key] = label_string
import numpy as np
import pandas as pd 
<load_from_csv>
prediction_dfs = []
row_id = list(prediction_dict.keys())
 birds = list(prediction_dict.values())
 prediction_df = pd.DataFrame({
"row_id": row_id,
"birds": birds
})
 prediction_dfs.append(prediction_df)
 prediction_df = pd.concat(prediction_dfs, axis=0, sort=False ).reset_index(drop=True)
 prediction_df.to_csv("submission.csv", index=False )
<sort_values><EOS> 
prediction_df['birds'].value_counts()