Kaggle competition: Cornell Birdcall Identification
<SOS> metric: MeanFScoreBeta Kaggle data source: cornell-birdcall-identification<compute_test_metric>
sys.path.append('.. /input/efficientnet-pytorch/EfficientNet-PyTorch/EfficientNet-PyTorch-master')
 warnings.filterwarnings("ignore", category=UserWarning)
 BASE_TEST_DIR = '.. /input/birdsong-recognition' if os.path.exists('.. /input/birdsong-recognition/test_audio')else '.. /input/my-birdcall-datasets'
valid_results = train_targets_scored.drop(columns=target_cols ).merge(train[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 y_true = train_targets_scored[target_cols].values
y_pred = valid_results[target_cols].values
score = 0
for i in range(len(target_cols)) :
score += log_loss(y_true[:, i], y_pred[:, i])
 print("CV log_loss: ", score / y_pred.shape[1] )<save_to_csv>
class Hparams() :
def __init__(self):
self.models_name = ['resnet50','efficientnet-b0','efficientnet-b0','efficientnet-b0','efficientnet-b0','resnet50']
self.chk = ['resnet50_78_0.830_0.666.pt','enet0_101_0.771_0.692.pt','enet0_45_0.558.pt','enet0_133_0.707_0.691.pt',
'150enet0_116_0.707_0.703.pt','2.5resnet50_113_0.715_0.693.pt']
self.count_bird = [265,265,265,265,150,265]
self.len_chack = [448,448,448,448,448,224]
self.mel_folder = './mel/'
self.n_fft = 892
self.sr = 21952
self.hop_length=245
self.n_mels = 224
self.win_length = self.n_fft
self.batch_size = 100
self.lr = 0.001
self.border = 0.5
self.save_interval = 200
self.bird_count = pd.read_csv('.. /input/my-birdcall-datasets/bird_count.csv' ).ebird_code.to_numpy()
self.BIRD_CODE = {b:i for i,b in enumerate(self.bird_count)}
self.INV_BIRD_CODE = {v: k for k, v in self.BIRD_CODE.items() }
self.bird_count = self.bird_count[:self.count_bird[0]]
hp = Hparams()
def mono_to_color(X: np.ndarray,len_chack, mean=0.5, std=0.5, eps=1e-6):
trans = transforms.Compose([transforms.ToPILImage() ,
transforms.Resize([hp.n_mels, len_chack]), transforms.ToTensor() ,
transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])])
 X = np.stack([X, X, X], axis=-1)
 V =(255 * X ).astype(np.uint8)
 V =(trans(V)+1)/2
return V
def accuracy(y_true, y_pred):
y_pred = torch.sigmoid(y_pred)
 y_pred = y_pred.detach().cpu().numpy()
return f1_score(y_true > hp.border, y_pred > hp.border, average="samples")
 def get_melspectr(train_path):
y, _ = librosa.load(train_path,sr=hp.sr,mono=True,res_type="kaiser_fast")
 spectr = librosa.feature.melspectrogram(y, sr=hp.sr, n_mels=hp.n_mels, n_fft=hp.n_fft, hop_length = hp.hop_length, win_length = hp.win_length, fmin = 300)
 return spectr.astype(np.float16)
 def random_power(images, power = 1.5, c= 0.7):
images = images - images.min()
images = images/(images.max() +0.0000001)
 images = images**(random.random() *power + c)
 return images
def test_accuracy(preds, log_stat= False, border=0.5):
answer = pd.read_csv('.. /input/my-birdcall-datasets/example_test_audio_summary.csv')
 preds = answer.merge(preds, how = 'right', left_on='filename_seconds', right_on='row_id')
 y_true, y_pred = [], []
my_bird = 0
pred_bird = 0
bad_bird = {}
for all in preds.loc[:,['bird','birds']].to_numpy() :
y = np.zeros(265)
 c = np.array(all[0].split())
 for bird in c:
y[hp.BIRD_CODE[bird]]=1
y_true.append(y)
 y = np.zeros(265)
 d = np.array(all[1].split())
 for bird in d:
y[hp.BIRD_CODE[bird]]=1
y_pred.append(y)
 mask = np.in1d(d, c)
 if d[0] != 'nocall':
pred_bird += len(d)
 if mask.sum() >0 and d[0] != 'nocall':
my_bird += mask.sum()
for i in d[~mask]:
if i in bad_bird:
bad_bird[i] += 1
else:
bad_bird[i] = 1
if not pred_bird: pred_bird = 1
f1 = f1_score(y_true, y_pred, average="samples")
 print("border: %.1f bird: %d bird_accuracy: %.3f test_accuracy: %.3f" %(
 border,my_bird, my_bird/pred_bird, f1))
if log_stat:
for w in sorted(bad_bird, key=bad_bird.get, reverse=True)[:5]:
print(w, bad_bird[w])
 return my_bird, my_bird/pred_bird, f1
class BirdcallNet(nn.Module):
def __init__(self, name, num_classes=265):
super(BirdcallNet, self ).__init__()
self.model = models.__getattribute__(name )(pretrained=False)
 if name in ["resnet50","resnext50_32x4d"]:
self.model.fc = nn.Linear(2048, num_classes)
 elif name in ['resnet18','resnet34']:
self.model.fc = nn.Linear(512, num_classes)
 elif name =="densenet121":
self.model.classifier = nn.Linear(1024, num_classes)
 elif name in ['alexnet','vgg16']:
self.model.classifier[-1] = nn.Linear(4096, num_classes)
 elif name =="mobilenet_v2":
self.model.classifier[1] = nn.Linear(1280, num_classes)
 def forward(self, x):
return self.model(x)
 def get_model(model_name,chk,count_bird):
best_bird_count,best_score, epochs = 0,0,1
all_loss, train_accuracy = [], []
f1_scores,t_scores,b_scores = [],[],[]
if not chk and model_name in ['efficientnet-b3','efficientnet-b0']:
model = EfficientNet.from_pretrained(model_name, num_classes = count_bird ).cuda()
optimizer = Adam(model.parameters() , lr = hp.lr)
 else:
models_names = ['alexnet','resnet50','resnet18','resnet34','mobilenet_v2','densenet121','resnext50_32x4d','densenet169']
if model_name in models_names:
model = BirdcallNet(model_name, hp.count_bird[0] ).cuda()
elif model_name == 'mini':
model = Classifier(hp.count_bird[0] ).cuda()
else:
model = EfficientNet.from_name(model_name, override_params={'num_classes': count_bird } ).cuda()
optimizer = Adam(model.parameters() , lr = hp.lr)
 if chk:
ckpt = torch.load('.. /input/my-birdcall-datasets/'+chk)
 model.load_state_dict(ckpt['model'])
 epochs = int(ckpt['epoch'])+ 1
train_accuracy = ckpt['train_accuracy']
all_loss = ckpt['all_loss']
best_bird_count = ckpt['best_bird_count']
best_score = ckpt['best_score']
if 'optimizer' in ckpt:
optimizer.load_state_dict(ckpt['optimizer'])
 if 't_scores' in ckpt:
t_scores = ckpt['t_scores']
if 'f1_scores' in ckpt:
f1_scores = ckpt['f1_scores']
if 'b_scores' in ckpt:
b_scores = ckpt['b_scores']
print('Чекпоинт загружен: Эпоха %d Число обнаруженых птиц %d Score %.3f' %(epochs,best_bird_count,best_score))
return model,optimizer, epochs, train_accuracy, all_loss, best_bird_count, best_score, t_scores, f1_scores, b_scores
<install_modules><EOS> 
def preprocess() :
os.makedirs(hp.mel_folder, exist_ok=True)
 dataset = glob.glob(os.path.join("./short/", '**/*.wav'), recursive=True)
 with open('meta.csv', 'w', encoding="utf-8")as output:
output.write("file,bird
")
 for train_path in tqdm(dataset):
wav_name = os.path.basename(train_path)
 dirname = train_path.split("\")[1]
mel = get_melspectr(train_path)
 mel = torch.from_numpy(mel)
 wav_path = os.path.join(hp.mel_folder, wav_name)
 save_path = wav_path.replace('.mp3', '.amp')
 torch.save(mel, save_path)
 output.write(wav_name +',' + dirname + "
")
 secondary_labels = pd.read_csv('train1.csv')
 meta = pd.read_csv('meta.csv')
 meta = meta.merge(secondary_labels[["filename",'labels_bg']], how = 'left', left_on='file', right_on='filename',copy=False)
 meta[["file","bird",'labels_bg']].to_csv('meta.csv', index=False)
 print('Training dataset created / Тренировочный датасет создан')
 class MelDataset(Dataset):
def __init__(self, bird_list, hp):
self.bird_list = bird_list
self.hp = hp
self.noise = pd.read_csv("nocall.csv")
 self.stop_border = 0.3
self.level_noise = 0.05
self.div_coef = 100
def __len__(self):
return len(self.bird_list)
 def __getitem__(self, idx):
idx2 = random.randint(0, len(self.bird_list)-1)
 idx3 = random.randint(0, len(self.bird_list)-1)
 y = torch.zeros(self.hp.count_bird[0])
 birds, background = [],[]
self.len_chack = random.randint(self.hp.len_chack[0]-48, self.hp.len_chack[0]+52)
 images = np.zeros(( self.hp.n_mels, self.len_chack)).astype(np.float32)
 for i,idy in enumerate([idx,idx2,idx3]):
sample = self.bird_list.loc[idy, :]
mel = torch.load(self.hp.mel_folder+sample.file.replace(".mp3",".amp")).numpy()
labels_bird = sample.bird.split()
for bird in labels_bird:
if not bird in birds and bird != 264:
birds.append(self.hp.BIRD_CODE[bird])
 if sample.labels_bg:
labels_bg = sample.labels_bg.split()
for bg in labels_bg:
if not bg in background:
background.append(self.hp.BIRD_CODE[bg])
 if mel.shape[1]>self.len_chack:
start = random.randint(0, mel.shape[1] - self.len_chack - 1)
 mel = mel[:, start : start + random.randint(self.len_chack-48, self.len_chack)]
else:
len_zero = random.randint(0, self.len_chack-mel.shape[1])
 mel = np.concatenate(( np.zeros(( self.hp.n_mels,len_zero)) ,mel), axis=1)
 mel = np.concatenate(( mel,np.zeros(( self.hp.n_mels,self.len_chack-mel.shape[1]))), axis=1)
 mel = random_power(mel, power = 3, c= 0.5)
 images = images + mel*(random.random() * self.div_coef + 1)
 if random.random() <self.stop_border:
break
idy = random.randint(0, len(self.noise)-1)
 sample = self.noise.loc[idy, :]
mel = torch.load('./mel/'+sample.file.replace(".wav",".amp")).numpy()
mel = np.concatenate(( np.zeros(( self.hp.n_mels,self.len_chack)) ,mel), axis=1)
 mel = np.concatenate(( mel,np.zeros(( self.hp.n_mels,self.len_chack))), axis=1)
 start = random.randint(0, mel.shape[1] - self.len_chack - 1)
 mel = mel[:, start : start + self.len_chack]
mel = random_power(mel)
 images = images + mel/(mel.max() +0.0000001)*(random.random() *1+0.5)*images.max()
images = librosa.power_to_db(images.astype(np.float32), ref=np.max)
 images =(images+80)/80
if random.random() <0.9:
images = images +(np.random.sample(( self.hp.n_mels,self.len_chack)).astype(np.float32)+9)* images.mean() * self.level_noise *(np.random.sample() + 0.3)
 if random.random() <0.9:
r = random.randint(1,self.hp.n_mels)
 pink_noise = np.array([np.concatenate(( 1 - np.arange(r)/r,np.zeros(self.hp.n_mels-r)))] ).T
images = images +(np.random.sample(( self.hp.n_mels,self.len_chack)).astype(np.float32)+9)* 2 * images.mean() * self.level_noise *(np.random.sample() + 0.3)
 if random.random() <0.9:
a = random.randint(0, self.hp.n_mels//2)
 b = random.randint(a+20, self.hp.n_mels)
 images[a:b,:] = images[a:b,:] +(np.random.sample(( b-a,self.len_chack)).astype(np.float32)+9)* 0.05 * images.mean() * self.level_noise *(np.random.sample() + 0.3)
 if random.random() <0.5:
images = images - images.min()
r = random.randint(self.hp.n_mels//2,self.hp.n_mels)
 x = random.random() /2
pink_noise = np.array([np.concatenate(( 1-np.arange(r)*x/r,np.zeros(self.hp.n_mels-r)-x+1)) ] ).T
images = images*pink_noise
images = images/images.max()
images = random_power(images, power = 2, c= 0.7)
 images = mono_to_color(images,hp.len_chack[0])
 if random.random() <0.0001:
img = images.numpy()
img = img - img.min()
img = img/img.max()
img = np.moveaxis(img, 0, 2)
 imgplot = plt.imshow(img)
 plt.savefig('log/img/'+("_".join(self.hp.INV_BIRD_CODE[x] for x in birds)) +'_'+sample.file+'.png')
 if not birds:
birds.append(264)
 for bird in background:
if bird < len(y):
y[bird]=0.3
for bird in birds:
y[bird]=1
return images, y
def train(model,optimizer,epochs,train_accuracy,all_loss,best_bird_count,best_score, t_scores, f1_scores, b_scores):
save_dir = os.path.join("./log")
 os.makedirs(save_dir, exist_ok=True)
 bird_list = pd.read_csv("meta.csv")
 bird_list = bird_list[bird_list.bird.isin(hp.bird_count)].reset_index(drop=True)
 bird_list = bird_list.fillna(0)
 train_count = len(bird_list)
 trainset = MelDataset(bird_list, hp)
 train_loader = data.DataLoader(trainset, batch_size = hp.batch_size, shuffle=True, drop_last=True, num_workers = 2)
 prediction_dict = {}
start = time.time()
model.zero_grad()
for epoch in range(epochs, 1000):
step = 0
model.train()
start_time = time.time()
for(mel, background)in train_loader:
step+=1
prediction = model(mel.cuda())
 train_loss = nn.BCEWithLogitsLoss()(prediction, background.cuda())
 train_loss.backward()
if not step %(100//hp.batch_size):
optimizer.step()
model.zero_grad()
train_accuracy.append(accuracy(background, prediction))
all_loss.append(train_loss.detach().cpu().numpy())
 if not step % hp.save_interval:
print(str(epoch)+' '+str(step)+'/'+str(train_count//hp.batch_size),
"время: %.3f loss: %.3f accuracy: %.3f " %(
(time.time() -start_time)/hp.save_interval,
np.mean(all_loss[-hp.save_interval:])*10,
np.mean(train_accuracy[-hp.save_interval:])))
(bird_count, bird_accuracy, test_accuracy), _ = generate([model],epochs,hp.border,True)
 if bird_accuracy>0:
t_scores.append(bird_accuracy)
 f1_scores.append(test_accuracy)
 b_scores.append(bird_count)
 model.train()
plt.clf()
plt.plot(gaussian_filter1d(train_accuracy[80:], 20))
plt.plot(gaussian_filter1d(all_loss[80:], 20)*10)
 plt.savefig('log/all_loss.png')
 plt.clf()
plt.plot(t_scores)
 plt.savefig('log/t.png')
 plt.clf()
plt.plot(f1_scores)
 plt.savefig('log/f1.png')
 plt.clf()
plt.plot(b_scores)
 plt.savefig('log/b.png')
 if(bird_count>best_bird_count or test_accuracy>best_score or step==hp.save_interval):
if bird_count>best_bird_count:
best_bird_count = bird_count
if test_accuracy>best_score:
best_score = test_accuracy
torch.save({
'model': model.state_dict() ,
'optimizer': optimizer.state_dict() ,
'epoch': epoch,
'best_bird_count': bird_count,
'best_score': test_accuracy,
'train_accuracy': train_accuracy,
'all_loss': all_loss,
't_scores': t_scores,
'f1_scores': f1_scores,
'b_scores': b_scores,
}, 'log/enet_%d_%.3f_%.3f.pt' %(bird_count,bird_accuracy,test_accuracy))
print("Модель сохранена")
 start_time = time.time()
def generate(models, epochs, border,log_stat):
start = time.time()
preds = []
TEST_FOLDER = f'{BASE_TEST_DIR}/test_audio/'
test_info = pd.read_csv(f'{BASE_TEST_DIR}/test.csv')
 unique_audio_id = test_info.audio_id.unique()
for model in models:
model.eval()
with torch.no_grad() :
for audio_id in unique_audio_id:
melspectr = get_melspectr(TEST_FOLDER + audio_id + ".mp3")
 melspectr = librosa.power_to_db(melspectr, amin=1e-7, ref=np.max)
 melspectr =(( melspectr+80)/80 ).astype(np.float16)
 test_df_for_audio_id = test_info.query(f"audio_id == '{audio_id}'" ).reset_index(drop=True)
 est_bird =np.zeros(( 265))
probass = {}
for index, row in test_df_for_audio_id.iterrows() :
site = row['site']
start_time = row['seconds'] - 5
row_id = row['row_id']
mels = []
probas = None
if site == 'site_1' or site == 'site_2':
start_index = int(hp.sr * start_time/hp.hop_length)
 end_index = int(hp.sr * row['seconds']/hp.hop_length)
 y = melspectr[:,start_index:end_index]
else:
y = melspectr
if(y.shape[1]%hp.len_chack[0]):
y = y[:,:-(y.shape[1]%448)]
prob = []
for i,model in enumerate(models):
mels = []
probas = None
ys = np.reshape(y,(hp.n_mels, -1, hp.len_chack[i]))
ys = np.moveaxis(ys, 1, 0)
 for image in ys:
image = image/image.max()
image = mono_to_color(image,hp.len_chack[i])
 mels.append(image)
 mels = np.stack(mels)
 for n in range(0,len(mels),hp.batch_size):
if len(mels)== 1:
mel = np.array(mels)
 else:
mel = mels[n:n+hp.batch_size]
mel = torch.from_numpy(mel ).cuda()
prediction = model(mel)
 prediction = torch.sigmoid(prediction)
 proba = prediction.detach().cpu().numpy()
proba = np.concatenate(( proba,np.zeros(( proba.shape[0],265-proba.shape[1]))), axis=1)
 if not probas is None:
probas = np.append(probas, proba, axis = 0)
 else:
probas = proba
if hp.len_chack[i] == 448:
probas = np.append(probas, proba, axis = 0)
 prob.append(probas)
 prob = np.stack(prob,axis=0)
 prob = prob**2
proba = prob.mean(axis=0)
 proba = proba**(1/2)
 for xx in proba:
z = xx.copy()
z[z<0.5] = 0
est_bird = est_bird + z/70
est_bird[(est_bird<0.15)&(est_bird>0)] = 0.15
probass[row_id] = proba
est_bird[est_bird>0.3] = 0.3
for row_id,probas in probass.items() :
prediction_dict = []
for proba in probas:
proba += est_bird
events = proba > border
labels = np.argwhere(events ).reshape(-1 ).tolist()
if len(labels)== 0 or(264 in labels):
continue
else:
labels_str_list = list(map(lambda x: hp.INV_BIRD_CODE[x], labels))
for i in labels_str_list:
if i not in prediction_dict:
prediction_dict.append(i)
 if len(prediction_dict)== 0:
prediction_dict = "nocall"
else:
prediction_dict = " ".join(prediction_dict)
 preds.append([row_id, prediction_dict])
 preds = pd.DataFrame(preds, columns=['row_id', 'birds'])
 preds.to_csv('submission.csv', index=False)
 return test_accuracy(preds,log_stat,border), time.time() - start
def pseudo(models):
files = {}
with open('meta_all.csv', 'r', encoding="utf-8")as input:
input.readline()
for s in input:
s = s.strip().split(',')
 file,bird,background = s[0],s[1],s[2].split(' ')
 file = file.split('.')[0]
files[file] = [bird,background]
dataset = glob.glob(os.path.join("./mel/", 'XC*.amp'), recursive=True)
 prediction_dict = {}
for model in models:
model.eval()
with torch.no_grad() :
for file_name in tqdm(dataset):
y = torch.load(file_name ).numpy()
est_bird =np.zeros(( 265))
mels = []
probas = None
ys = []
if y.shape[1]>=hp.len_chack:
for i in range(0,(y.shape[1]*5)//hp.len_chack-4,3):
yy = y[:,i*hp.len_chack//5:(i+5)*hp.len_chack//5]
if yy.shape[1]<hp.len_chack:
yy = np.concatenate(( yy,np.zeros(( hp.n_mels,hp.len_chack-yy.shape[1]))), axis=1)
 ys.append(yy)
 ys = np.stack(ys)
 else:
yy = np.concatenate(( y,np.zeros(( hp.n_mels,hp.len_chack-y.shape[1]))), axis=1)
 ys = np.array([yy])
 for y in ys:
y = librosa.power_to_db(y, amin=1e-7, ref=np.max)
 y =(( y+80)/80 ).astype(np.float16)
 image = torch.from_numpy(np.stack([y, y, y])).float()
mels.append(image)
 mels = np.stack(mels)
 for n in range(0,len(mels),hp.batch_size*2):
if len(mels)== 1:
mel = np.array(mels)
 else:
mel = mels[n:n+hp.batch_size*2]
mel = torch.from_numpy(mel ).cuda()
prob = []
for model in models:
prediction = model(mel)
 prediction = torch.sigmoid(prediction)
 proba = prediction.detach().cpu().numpy()
proba = np.concatenate(( proba,np.zeros(( proba.shape[0],265-proba.shape[1]))), axis=1)
 prob.append(proba)
 prob = np.stack(prob)
 proba = prob.mean(axis=0)
 if not probas is None:
probas = np.append(probas, proba, axis = 0)
 else:
probas = proba
for sk,proba in enumerate(probas):
proba += est_bird
file_name1 = os.path.basename(file_name ).replace('.mp3','' ).replace('.wav','' ).replace('.amp','')
 sek = file_name1 +'_' + str(sk)+'.amp'
events = proba > hp.border
labels = np.argwhere(events ).reshape(-1 ).tolist()
if len(labels)== 0 or(264 in labels):
continue
else:
labels_str_list = list(map(lambda x: hp.INV_BIRD_CODE[x], labels))
if file_name1 in files:
bird = []
background = []
for i in labels_str_list:
if i in files[file_name1][1] or i==files[file_name1][0]:
if not i in bird:
bird.append(i)
 else:
if not i in background:
background.append(i)
 if bird:
torch.save(torch.from_numpy(ys[sk].astype(np.float16)) , './mel_p/'+sek)
 prediction_dict[sek] = [" ".join(bird)," ".join(background)]

 preds = pd.DataFrame.from_dict(prediction_dict, orient='index', columns=['birds',"labels_bg"])
 preds.reset_index(inplace=True)
 preds.columns = ['file', 'bird',"labels_bg"]
preds.to_csv('meta.csv', index=False)
 hp = Hparams()
all_model = []
for i in range(len(hp.models_name)) :
model,optimizer, epochs, train_accuracy, all_loss, best_bird_count, best_score, t_scores, f1_scores, b_scores = get_model(
 hp.models_name[i],hp.chk[i],hp.count_bird[i])
 all_model.append(model)
 generate(all_model, epochs, hp.border, True)