Kaggle competition: Cornell Birdcall Identification
<SOS> metric: MeanFScoreBeta Kaggle data source: cornell-birdcall-identification<define_variables>
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS = 24
BATCH_SIZE = 128
WEIGHT_DECAY = {'ALL_TARGETS': 1e-5, 'SCORED_ONLY': 3e-6}
MAX_LR = {'ALL_TARGETS': 1e-2, 'SCORED_ONLY': 3e-3}
DIV_FACTOR = {'ALL_TARGETS': 1e3, 'SCORED_ONLY': 1e2}
PCT_START = 0.1
SEEDS = 7
NFOLDS = 7
DRUG_THRESH = 18
feature_cols = [c for c in train.columns if c not in all_target_cols]
feature_cols = [c for c in feature_cols if c not in ['kfold', 'sig_id', 'drug_id']]
num_features = len(feature_cols)
 model = Model(num_features, num_all_targets)
 def make_cv_folds(train, SEEDS, NFOLDS, DRUG_THRESH):
vc = train.drug_id.value_counts()
vc1 = vc.loc[vc <= DRUG_THRESH].index.sort_values()
vc2 = vc.loc[vc > DRUG_THRESH].index.sort_values()
for seed_id in range(SEEDS):
kfold_col = 'kfold_{}'.format(seed_id)
 dct1 = {}
dct2 = {}
skf = MultilabelStratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=seed_id)
 tmp = train.groupby('drug_id')[target_cols].mean().loc[vc1]
for fold,(idxT, idxV)in enumerate(skf.split(tmp, tmp[target_cols])) :
dd = {k: fold for k in tmp.index[idxV].values}
dct1.update(dd)
 skf = MultilabelStratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=seed_id)
 tmp = train.loc[train.drug_id.isin(vc2)].reset_index(drop=True)
 for fold,(idxT, idxV)in enumerate(skf.split(tmp, tmp[target_cols])) :
dd = {k: fold for k in tmp.sig_id[idxV].values}
dct2.update(dd)
 train[kfold_col] = train.drug_id.map(dct1)
 train.loc[train[kfold_col].isna() , kfold_col] = train.loc[train[kfold_col].isna() , 'sig_id'].map(dct2)
 train[kfold_col] = train[kfold_col].astype('int8')
 return train
train = make_cv_folds(train, SEEDS, NFOLDS, DRUG_THRESH)
 display(train.head())
 def run_training(fold_id, seed_id):
seed_everything(seed_id)
 train_ = train
test_ = test
kfold_col = f'kfold_{seed_id}'
trn_idx = train_[train_[kfold_col] != fold_id].index
val_idx = train_[train_[kfold_col] == fold_id].index
train_df = train_[train_[kfold_col] != fold_id].reset_index(drop=True)
 valid_df = train_[train_[kfold_col] == fold_id].reset_index(drop=True)
 model = Model(num_features, num_targets)
 model.load_state_dict(torch.load(f".. /input/nn-transfer-learning-oof/SCORED_ONLY_SEED_{seed_id}_FOLD{fold_id}_.pth"))
model.to(DEVICE)
 x_test = test_[feature_cols].values
testdataset = TestDataset(x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 predictions = np.zeros(( len(test_), num_targets))
predictions = inference_fn(model, testloader, DEVICE)
 print(f"Pretrained_Model_Loaded_SCORED_ONLYSEED: {seed_id}, FOLD: {fold_id} ")
 return 0.02, predictions<categorify>
!pip install -q tensorflow_io
!pip install -q tfp-nightly
def run_k_fold(NFOLDS, seed_id):
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for fold_id in range(NFOLDS):
oof_, pred_ = run_training(fold_id, seed_id)
 predictions += pred_ / NFOLDS
oof += oof_
return oof, predictions
SEED = [0, 1, 2, 3, 4, 5, 6]
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 time_begin = time()
for seed_id in SEED:
oof_, predictions_ = run_k_fold(NFOLDS, seed_id)
 oof += oof_ / len(SEED)
 predictions += predictions_ / len(SEED)
 time_diff = time() - time_begin
train[target_cols] = oof
test[target_cols] = predictions
str(timedelta(seconds=time_diff))
data_dir = '.. /input/lish-moa/'
train_targets_scored = pd.read_csv(data_dir + 'train_targets_scored.csv')
 train_targets_scored.head()
valid_results = train_targets_scored.drop(columns=target_cols ).merge(train[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 y_true = train_targets_scored[target_cols].values
y_pred = valid_results[target_cols].values
score = 0
for i in range(len(target_cols)) :
score += log_loss(y_true[:, i], y_pred[:, i])
 print("CV log_loss: ", score / y_pred.shape[1])
 sample_submission = pd.read_csv(data_dir + 'sample_submission.csv')
 sub_nn_transfer = sample_submission.drop(columns=target_cols ).merge(test[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 sub_nn_transfer.to_csv('submission_nn_transfer.csv', index=False )<load_pretrained>
!pip install cloud-tpu-client
print(tf.__version__)
 Client().configure_tpu_version(tf.__version__, restart_type='ifNeeded' )
saved_path_name = '.. /input/nn-transfer-learning-oof/'
oof_nn_transfer = np.load(saved_path_name + 'oof_nn_transfer_all.npy')
 oof_nn_transfer.shape<load_from_csv>
print(tf.__version__ )
def readin_data_resnet() :
train_features = pd.read_csv('/kaggle/input/lish-moa/train_features.csv')
 train_drug = pd.read_csv('.. /input/lish-moa/train_drug.csv')
 non_ctl_idx = train_features.loc[train_features['cp_type']!='ctl_vehicle'].index.to_list()
train_features = train_features.drop(['sig_id','cp_type','cp_dose','cp_time'],axis=1)
 train_targets_scored = pd.read_csv('/kaggle/input/lish-moa/train_targets_scored.csv')
 targets_drug = train_targets_scored.merge(train_drug, on='sig_id')
 train_targets_scored = train_targets_scored.drop('sig_id',axis=1)
 labels_train = train_targets_scored.values
train_features = train_features.iloc[non_ctl_idx]
labels_train = labels_train[non_ctl_idx]
targets_drug = targets_drug.iloc[non_ctl_idx]
test_features = pd.read_csv('/kaggle/input/lish-moa/test_features.csv')
 test_features = test_features.drop(['sig_id','cp_dose','cp_time'],axis=1)
 json_file_path = '.. /input/ttestpcarfelogisticregression/main_predictors.json'
with open(json_file_path, 'r')as j:
predictors = json.loads(j.read())
 predictors = predictors['start_predictors']
cs = train_features.columns.str.contains('c-')
 gs = train_features.columns.str.contains('g-')
 return train_features,labels_train,test_features,targets_drug,predictors,cs,gs 
<compute_test_metric>
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
AUTO = tf.data.experimental.AUTOTUNE
BATCH_SIZE = 32 * strategy.num_replicas_in_sync
def logloss(y_true, y_pred):
y_pred = tf.clip_by_value(y_pred,p_min,p_max)
 return -backend.mean(y_true*backend.log(y_pred)+(1-y_true)*backend.log(1-y_pred))
def resnet_preprocessor(train,test,cs,gs):
n_gs = 2
n_cs = 100
pca_cs = PCA(n_components = n_cs)
 pca_gs = PCA(n_components = n_gs)
 train_pca_gs = pca_gs.fit_transform(train[:,gs])
 train_pca_cs = pca_cs.fit_transform(train[:,cs])
 test_pca_gs = pca_gs.transform(test[:,gs])
 test_pca_cs = pca_cs.transform(test[:,cs])
 train_c_mean = train[:,cs].mean(axis=1)
 test_c_mean = test[:,cs].mean(axis=1)
 train_g_mean = train[:,gs].mean(axis=1)
 test_g_mean = test[:,gs].mean(axis=1)
 train = np.concatenate(( train,train_pca_gs,train_pca_cs,train_c_mean[:,np.newaxis]
,train_g_mean[:,np.newaxis]),axis=1)
 test = np.concatenate(( test,test_pca_gs,test_pca_cs,test_c_mean[:,np.newaxis],
test_g_mean[:,np.newaxis]),axis=1)
 scaler = preprocessing.QuantileTransformer(n_quantiles=100,random_state=0, output_distribution="normal")
 train = scaler.fit_transform(train)
 test = scaler.transform(test)
 return train, test 
<choose_model_class>
SEED = 777
os.environ['PYTHONHASHSEED'] = str(SEED)
 np.random.seed(SEED)
 tf.random.set_seed(SEED)
 USE_FOLD=2
N_SPLIT=7
SAMPLE_RATE=32000
NUM_CLASSES = 264
PERIOD = 5
def resnet_model(n_features, n_features_2, n_labels, label_smoothing = 0.0005):
input_1 = layers.Input(shape =(n_features,), name = 'Input1')
 input_2 = layers.Input(shape =(n_features_2,), name = 'Input2')
 head_1 = Sequential([
layers.BatchNormalization() ,
layers.Dropout(0.25),
layers.Dense(512, activation="elu"),
layers.BatchNormalization() ,
layers.Dense(256, activation = "elu")
 ],name='Head1')
 input_3 = head_1(input_1)
 input_3_concat = layers.Concatenate()([input_2, input_3])
 head_2 = Sequential([
layers.BatchNormalization() ,
layers.Dropout(0.3),
layers.Dense(512, "relu"),
layers.BatchNormalization() ,
layers.Dense(512, "elu"),
layers.BatchNormalization() ,
layers.Dense(256, "relu"),
layers.BatchNormalization() ,
layers.Dense(256, "elu")
 ],name='Head2')
 input_4 = head_2(input_3_concat)
 input_4_avg = layers.Average()([input_3, input_4])
 head_3 = Sequential([
layers.BatchNormalization() ,
layers.Dense(256, kernel_initializer='lecun_normal', activation='selu'),
layers.BatchNormalization() ,
layers.Dense(n_labels, kernel_initializer='lecun_normal', activation='selu'),
layers.BatchNormalization() ,
layers.Dense(n_labels, activation="sigmoid")
 ],name='Head3')
 output = head_3(input_4_avg)
 model = Model(inputs = [input_1, input_2], outputs = output)
 model.compile(optimizer='adam', loss=losses.BinaryCrossentropy(label_smoothing=label_smoothing), metrics=logloss)
 return model<sort_values>
train = pd.read_csv('.. /input/birdsong-resampled-train-audio-00/train_mod.csv' )
def resnet_make_cv_folds(targets_drug,target_cols, SEEDS, NFOLDS, DRUG_THRESH):
vc = targets_drug.drug_id.value_counts()
vc1 = vc.loc[vc <= DRUG_THRESH].index.sort_values()
vc2 = vc.loc[vc > DRUG_THRESH].index.sort_values()
for seed_id in SEEDS:
kfold_col = 'kfold_{}'.format(seed_id)
 dct1 = {}
dct2 = {}
skf = MultilabelStratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=seed_id)
 tmp = targets_drug.groupby('drug_id')[target_cols].mean().loc[vc1]
for fold,(idxT, idxV)in enumerate(skf.split(tmp, tmp[target_cols])) :
dd = {k: fold for k in tmp.index[idxV].values}
dct1.update(dd)
 skf = MultilabelStratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=seed_id)
 tmp = targets_drug.loc[targets_drug.drug_id.isin(vc2)].reset_index(drop=True)
 for fold,(idxT, idxV)in enumerate(skf.split(tmp, tmp[target_cols])) :
dd = {k: fold for k in tmp.sig_id[idxV].values}
dct2.update(dd)
 targets_drug[kfold_col] = targets_drug.drug_id.map(dct1)
 targets_drug.loc[targets_drug[kfold_col].isna() , kfold_col] = targets_drug.loc[targets_drug[kfold_col].isna() , 'sig_id'].map(dct2)
 targets_drug[kfold_col] = targets_drug[kfold_col].astype('int8')
 return targets_drug 
<predict_on_test>
BIRD_CODE = {
'aldfly': 0, 'ameavo': 1, 'amebit': 2, 'amecro': 3, 'amegfi': 4,
'amekes': 5, 'amepip': 6, 'amered': 7, 'amerob': 8, 'amewig': 9,
'amewoo': 10, 'amtspa': 11, 'annhum': 12, 'astfly': 13, 'baisan': 14,
'baleag': 15, 'balori': 16, 'banswa': 17, 'barswa': 18, 'bawwar': 19,
'belkin1': 20, 'belspa2': 21, 'bewwre': 22, 'bkbcuc': 23, 'bkbmag1': 24,
'bkbwar': 25, 'bkcchi': 26, 'bkchum': 27, 'bkhgro': 28, 'bkpwar': 29,
'bktspa': 30, 'blkpho': 31, 'blugrb1': 32, 'blujay': 33, 'bnhcow': 34,
'boboli': 35, 'bongul': 36, 'brdowl': 37, 'brebla': 38, 'brespa': 39,
'brncre': 40, 'brnthr': 41, 'brthum': 42, 'brwhaw': 43, 'btbwar': 44,
'btnwar': 45, 'btywar': 46, 'buffle': 47, 'buggna': 48, 'buhvir': 49,
'bulori': 50, 'bushti': 51, 'buwtea': 52, 'buwwar': 53, 'cacwre': 54,
'calgul': 55, 'calqua': 56, 'camwar': 57, 'cangoo': 58, 'canwar': 59,
'canwre': 60, 'carwre': 61, 'casfin': 62, 'caster1': 63, 'casvir': 64,
'cedwax': 65, 'chispa': 66, 'chiswi': 67, 'chswar': 68, 'chukar': 69,
'clanut': 70, 'cliswa': 71, 'comgol': 72, 'comgra': 73, 'comloo': 74,
'commer': 75, 'comnig': 76, 'comrav': 77, 'comred': 78, 'comter': 79,
'comyel': 80, 'coohaw': 81, 'coshum': 82, 'cowscj1': 83, 'daejun': 84,
'doccor': 85, 'dowwoo': 86, 'dusfly': 87, 'eargre': 88, 'easblu': 89,
'easkin': 90, 'easmea': 91, 'easpho': 92, 'eastow': 93, 'eawpew': 94,
'eucdov': 95, 'eursta': 96, 'evegro': 97, 'fiespa': 98, 'fiscro': 99,
'foxspa': 100, 'gadwal': 101, 'gcrfin': 102, 'gnttow': 103, 'gnwtea': 104,
'gockin': 105, 'gocspa': 106, 'goleag': 107, 'grbher3': 108, 'grcfly': 109,
'greegr': 110, 'greroa': 111, 'greyel': 112, 'grhowl': 113, 'grnher': 114,
'grtgra': 115, 'grycat': 116, 'gryfly': 117, 'haiwoo': 118, 'hamfly': 119,
'hergul': 120, 'herthr': 121, 'hoomer': 122, 'hoowar': 123, 'horgre': 124,
'horlar': 125, 'houfin': 126, 'houspa': 127, 'houwre': 128, 'indbun': 129,
'juntit1': 130, 'killde': 131, 'labwoo': 132, 'larspa': 133, 'lazbun': 134,
'leabit': 135, 'leafly': 136, 'leasan': 137, 'lecthr': 138, 'lesgol': 139,
'lesnig': 140, 'lesyel': 141, 'lewwoo': 142, 'linspa': 143, 'lobcur': 144,
'lobdow': 145, 'logshr': 146, 'lotduc': 147, 'louwat': 148, 'macwar': 149,
'magwar': 150, 'mallar3': 151, 'marwre': 152, 'merlin': 153, 'moublu': 154,
'mouchi': 155, 'moudov': 156, 'norcar': 157, 'norfli': 158, 'norhar2': 159,
'normoc': 160, 'norpar': 161, 'norpin': 162, 'norsho': 163, 'norwat': 164,
'nrwswa': 165, 'nutwoo': 166, 'olsfly': 167, 'orcwar': 168, 'osprey': 169,
'ovenbi1': 170, 'palwar': 171, 'pasfly': 172, 'pecsan': 173, 'perfal': 174,
'phaino': 175, 'pibgre': 176, 'pilwoo': 177, 'pingro': 178, 'pinjay': 179,
'pinsis': 180, 'pinwar': 181, 'plsvir': 182, 'prawar': 183, 'purfin': 184,
'pygnut': 185, 'rebmer': 186, 'rebnut': 187, 'rebsap': 188, 'rebwoo': 189,
'redcro': 190, 'redhea': 191, 'reevir1': 192, 'renpha': 193, 'reshaw': 194,
'rethaw': 195, 'rewbla': 196, 'ribgul': 197, 'rinduc': 198, 'robgro': 199,
'rocpig': 200, 'rocwre': 201, 'rthhum': 202, 'ruckin': 203, 'rudduc': 204,
'rufgro': 205, 'rufhum': 206, 'rusbla': 207, 'sagspa1': 208, 'sagthr': 209,
'savspa': 210, 'saypho': 211, 'scatan': 212, 'scoori': 213, 'semplo': 214,
'semsan': 215, 'sheowl': 216, 'shshaw': 217, 'snobun': 218, 'snogoo': 219,
'solsan': 220, 'sonspa': 221, 'sora': 222, 'sposan': 223, 'spotow': 224,
'stejay': 225, 'swahaw': 226, 'swaspa': 227, 'swathr': 228, 'treswa': 229,
'truswa': 230, 'tuftit': 231, 'tunswa': 232, 'veery': 233, 'vesspa': 234,
'vigswa': 235, 'warvir': 236, 'wesblu': 237, 'wesgre': 238, 'weskin': 239,
'wesmea': 240, 'wessan': 241, 'westan': 242, 'wewpew': 243, 'whbnut': 244,
'whcspa': 245, 'whfibi': 246, 'whtspa': 247, 'whtswi': 248, 'wilfly': 249,
'wilsni1': 250, 'wiltur': 251, 'winwre3': 252, 'wlswar': 253, 'wooduc': 254,
'wooscj2': 255, 'woothr': 256, 'y00475': 257, 'yebfly': 258, 'yebsap': 259,
'yehbla': 260, 'yelwar': 261, 'yerwar': 262, 'yetvir': 263
}
INV_BIRD_CODE = {v: k for k, v in BIRD_CODE.items() }
def predict_resnet(train_features,test_features,labels_train, targets_drug1,SEEDS, NFOLDS,cs,gs):
n_labels = labels_train.shape[1]
n_train = train_features.shape[0]
n_test = test_features.shape[0]
p_min = 0.0005
p_max = 0.9995
n_seeds = len(SEEDS)
 n_folds = NFOLDS
y_pred = np.zeros(( n_test,n_labels))
oof = np.zeros(( n_train,n_labels))
train_all,data_test = resnet_preprocessor(train_features.values,
test_features.drop('cp_type',axis=1 ).values,cs,gs)
 for seed in SEEDS:
kfold_col = 'kfold_{}'.format(seed)
 kf = targets_drug1[kfold_col]
for fold in range(NFOLDS):
train = targets_drug1[kfold_col] != fold
test = targets_drug1[kfold_col] == fold
X_train, X_test = train_all[train],train_all[test]
X_train_2 = train_features[train][predictors].values
X_test_2 = train_features[test][predictors].values
data_test_2 = test_features[predictors].values
y_train = labels_train[train]
y_test = labels_train[test]
n_features = X_train.shape[1]
n_features_2 = X_train_2.shape[1]
model = keras.models.load_model('/kaggle/input/multi-input-resnet-model/TwoHeads_seed_'+str(seed)+'_fold_'+str(fold), custom_objects={'logloss':logloss})
 y_val = model.predict([X_test,X_test_2])
 oof[test] += y_val/ n_seeds
y_pred += model.predict([data_test,data_test_2])/(n_folds*n_seeds)
 return oof,y_pred 
<find_best_model_class>
SPS_CODE = dict(zip(train.sci_name, train.ebird_code))
def get_sps_from_bg(s):
if type(s)!= str: s = str(s)
 return re.findall('\((.*?)\)', s)
 def get_sps_from_sec(l):
return [re.split('_', s[1:-1])[0] for s in re.split(',\s*', l[1:-1])]
def map_sps_to_code(l):
res = []
for s in l:
if s == '':
continue
elif s in SPS_CODE:
res.append(SPS_CODE[s])
 else:
None
return res
train['labels_bg'] = train['background'].apply(get_sps_from_bg ).apply(map_sps_to_code)
 train['labels_sec'] = train['secondary_labels'].apply(get_sps_from_sec ).apply(map_sps_to_code)
 train['secondary_codes'] = [list(set(a+b)) for a,b in zip(train['labels_sec'].values, train['labels_bg'].values)]
train_features,labels_train,test_features,targets_drug,predictors,cs,gs = readin_data_resnet()
p_min = 0.0005
p_max = 0.9995
DRUG_THRESH = 18
n_seeds = 7
np.random.seed(42)
 SEEDS = np.random.randint(0,100,size=n_seeds)
 NFOLDS = 10
target_cols = targets_drug.drop(['sig_id','drug_id'], axis=1 ).columns.values.tolist()
targets_drug_copy = targets_drug.copy()
targets_drug1 = resnet_make_cv_folds(targets_drug_copy,target_cols, SEEDS, NFOLDS, DRUG_THRESH)
 oof_rs, test_pred_rs = predict_resnet(train_features,test_features,labels_train, targets_drug1,SEEDS, NFOLDS,cs,gs )<save_to_csv>
filter_limit = 30
train['secondary_codes'] = train['secondary_codes'].where(train['duration'] < filter_limit, '[]')
 train.secondary_codes = train.secondary_codes.apply(lambda x: list(x for x in map(BIRD_CODE.get, x)if x is not None))
sub_rs = pd.read_csv('/kaggle/input/lish-moa/sample_submission.csv')
 sub_rs.iloc[:,1:] = np.clip(test_pred_rs,p_min,p_max)
 sub_rs.iloc[test_features['cp_type'] == 'ctl_vehicle',1:] = 0
sub_rs.to_csv('submission_rs.csv', index=False )<compute_train_metric>
def downsampling(df):
main_data = pd.DataFrame()
for ebird_code in list(df.groupby('ebird_code' ).groups.keys()):
main_data = pd.concat([main_data, df[df['ebird_code'] == ebird_code].sort_values('duration')[:80]])
 return main_data
def upsampling(df):
def duplicate_long_data(df, ebird_code, dif):
long_data = df[df['ebird_code'] == ebird_code].query('duration > 40' ).sort_values('duration', ascending=False)[:dif]
df = pd.concat([df, long_data])
 return df
for i in range(5):
ebird_code_count = df.groupby('ebird_code' ).count().filename
for ebird_code in ebird_code_count.keys() :
dif = 60 - ebird_code_count[ebird_code]
if dif > 0:
df = duplicate_long_data(df, ebird_code, dif)
 else:
continue
return df
def log_loss_numpy(y_pred):
y_true_ravel = np.asarray(y_true ).ravel()
y_pred = np.asarray(y_pred ).ravel()
y_pred = np.clip(y_pred, 1e-15, 1 - 1e-15)
 loss = np.where(y_true_ravel == 1, - np.log(y_pred), - np.log(1 - y_pred))
return loss.mean()
def func_numpy_metric(weights):
oof_blend = np.tensordot(weights, oof, axes =(( 0),(0)))
 return log_loss_numpy(oof_blend)
 def grad_func(weights):
oof_clip = np.clip(oof, 1e-15, 1 - 1e-15)
 gradients = np.zeros(oof.shape[0])
 for i in range(oof.shape[0]):
a, b, c = y_true, oof_clip[i], np.zeros(( oof.shape[1], oof.shape[2]))
for j in range(oof.shape[0]):
if j != i:
c += weights[j] * oof_clip[j]
gradients[i] = -np.mean(( -a*b+(b**2)*weights[i]+b*c)/(( b**2)*(weights[i]**2)+2*b*c*weights[i]-b*weights[i]+(c**2)-c))
return gradients
@njit
def grad_func_jit(weights):
oof_clip = np.minimum(1 - 1e-15, np.maximum(oof, 1e-15))
gradients = np.zeros(oof.shape[0])
 for i in range(oof.shape[0]):
a, b, c = y_true, oof_clip[i], np.zeros(( oof.shape[1], oof.shape[2]))
for j in range(oof.shape[0]):
if j != i:
c += weights[j] * oof_clip[j]
gradients[i] = -np.mean(( -a*b+(b**2)*weights[i]+b*c)/(( b**2)*(weights[i]**2)+2*b*c*weights[i]-b*weights[i]+(c**2)-c))
return gradients<load_from_csv>
GCS_PATH0 = KaggleDatasets().get_gcs_path('birdsong-resampled-train-audio-00')
 GCS_PATH1 = KaggleDatasets().get_gcs_path('birdsong-resampled-train-audio-01')
 GCS_PATH2 = KaggleDatasets().get_gcs_path('birdsong-resampled-train-audio-02')
 GCS_PATH3 = KaggleDatasets().get_gcs_path('birdsong-resampled-train-audio-03')
 GCS_PATH4 = KaggleDatasets().get_gcs_path('birdsong-resampled-train-audio-04')
 train_all = train.copy()
train_all['file_path'] = ''
def f(x):
fl = x[0]
def get_gcp_path(fl):
if re.compile('[a-b]' ).fullmatch(fl):
return GCS_PATH0
elif re.compile('[c-f]' ).fullmatch(fl):
return GCS_PATH1
elif re.compile('[g-m]' ).fullmatch(fl):
return GCS_PATH2
elif re.compile('[n-r]' ).fullmatch(fl):
return GCS_PATH3
elif re.compile('[s-y]' ).fullmatch(fl):
return GCS_PATH4
else:
raise Exception('no matching code')
 return get_gcp_path(fl)+ '/' + x + '/'
train_all['file_path'] = train_all['ebird_code'].map(f)+ train_all['resampled_filename']
train_all = downsampling(train_all)
 skf = StratifiedKFold(n_splits=N_SPLIT, shuffle=True, random_state=1)
 train_all["fold"] = -1
for fold_id,(train_index, val_index)in enumerate(skf.split(train_all, train_all["ebird_code"])) :
train_all.iloc[val_index, -1] = fold_id
train_all = upsampling(train_all)
 use_fold = USE_FOLD
train_file_list = train_all.query("fold != @use_fold")[["file_path", "ebird_code", "secondary_codes"]]
val_file_list = train_all.query("fold == @use_fold")[["file_path", "ebird_code", "secondary_codes"]]
print("[fold {}] train: {}, val: {}".format(use_fold, len(train_file_list), len(val_file_list)) )
y_true = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv', index_col = 'sig_id' ).values
oof_dict = {'Model 1': '.. /input/nn-transfer-learning-oof/oof_nn_transfer_all.npy',
'Model 2': '.. /input/tabnet-oof-seed3-fold7-112220/oof_TabNet_all.npy',
'Resnet': '.. /input/prediction/resnet_oof.npy'
}
oof = np.zeros(( len(oof_dict), y_true.shape[0], y_true.shape[1]))
for i in range(oof.shape[0]):
oof[i] = np.load(list(oof_dict.values())[i] )<compute_test_metric>
train_file_list.ebird_code = train_file_list.ebird_code.map(BIRD_CODE)
 val_file_list.ebird_code = val_file_list.ebird_code.map(BIRD_CODE)
 train_file_list.secondary_codes = [list(set(s + [p])) for s , p in zip(train_file_list.secondary_codes, train_file_list.ebird_code)]
val_file_list.secondary_codes = [list(set(s + [p])) for s , p in zip(val_file_list.secondary_codes, val_file_list.ebird_code)]
%%time
log_loss_scores = {}
for n, key in enumerate(oof_dict.keys()):
score_oof = log_loss_numpy(oof[n])
 log_loss_scores[key] = score_oof
print(f'{key} CV:\t', score_oof)
 print('-' * 50 )<define_variables>
def mixup(batch_size, alpha, images, labels):

 if tf.random.uniform([])< 0.5:
return images,(tf.zeros([batch_size, 1, NUM_CLASSES]),labels)
 mix_weight = tfp.distributions.Beta(alpha, alpha ).sample([batch_size, 1])
 mix_weight = tf.maximum(mix_weight, 1.- mix_weight)
 images_mix_weight = tf.reshape(mix_weight, [batch_size, 1, 1, 1])
 images_mix =(
 images * images_mix_weight + images[::-1] *(1.- images_mix_weight))
labels_mix = labels * mix_weight + labels[::-1] *(1.- mix_weight)
 return images_mix,(tf.zeros([batch_size, 1, NUM_CLASSES]),labels_mix)
 def augment_freq_time_mask(spectrogram,
frequency_masking_para=15,
time_masking_para=15,
frequency_mask_num=1,
time_mask_num=1):
time_max = tf.shape(spectrogram)[1]
freq_max = tf.shape(spectrogram)[2]
for _ in range(frequency_mask_num):
f = tf.random.uniform(shape=() , minval=0, maxval=frequency_masking_para, dtype=tf.dtypes.int32)
 f0 = tf.random.uniform(shape=() , minval=0, maxval=freq_max - f, dtype=tf.dtypes.int32)
 value_ones_freq_prev = tf.ones(shape=[1, time_max, f0])
 value_zeros_freq = tf.zeros(shape=[1, time_max, f])
 value_ones_freq_next = tf.ones(shape=[1, time_max, freq_max-(f0+f)])
 freq_mask = tf.concat([value_ones_freq_prev, value_zeros_freq, value_ones_freq_next], axis=2)
 spectrogram = spectrogram*freq_mask
for _ in range(time_mask_num):
t = tf.random.uniform(shape=() , minval=0, maxval=time_masking_para, dtype=tf.dtypes.int32)
 t0 = tf.random.uniform(shape=() , minval=0, maxval=time_max - t, dtype=tf.dtypes.int32)
 value_zeros_time_prev = tf.ones(shape=[1, t0, freq_max])
 value_zeros_time = tf.zeros(shape=[1, t, freq_max])
 value_zeros_time_next = tf.ones(shape=[1, time_max-(t0+t), freq_max])
 time_mask = tf.concat([value_zeros_time_prev, value_zeros_time, value_zeros_time_next], axis=1)
 spectrogram = spectrogram*time_mask
return spectrogram
def augment_pitch_and_tempo(spectrogram,
max_tempo=1.2,
max_pitch=1.1,
min_pitch=0.95):
original_shape = tf.shape(spectrogram)
 choosen_pitch = tf.random.uniform(shape=() , minval=min_pitch, maxval=max_pitch)
 choosen_tempo = tf.random.uniform(shape=() , minval=1, maxval=max_tempo)
 new_freq_size = tf.cast(tf.cast(original_shape[2], tf.float32)*choosen_pitch, tf.int32)
 new_time_size = tf.cast(tf.cast(original_shape[1], tf.float32)/(choosen_tempo), tf.int32)
 spectrogram_aug = tf.image.resize(tf.expand_dims(spectrogram, -1), [new_time_size, new_freq_size])
 spectrogram_aug = tf.image.crop_to_bounding_box(spectrogram_aug, offset_height=0, offset_width=0, target_height=tf.shape(spectrogram_aug)[1], target_width=tf.minimum(original_shape[2], new_freq_size))
spectrogram_aug = tf.cond(choosen_pitch < 1,
lambda: tf.image.pad_to_bounding_box(spectrogram_aug, offset_height=0, offset_width=0,
target_height=tf.shape(spectrogram_aug)[1], target_width=original_shape[2]),
lambda: spectrogram_aug)
 return spectrogram_aug[:, :, :, 0]
def augment_speed_up(spectrogram,
speed_std=0.1):
original_shape = tf.shape(spectrogram)
 choosen_speed = tf.math.abs(tf.random.normal(shape=() , stddev=speed_std))
choosen_speed = 1 + choosen_speed
new_freq_size = tf.cast(tf.cast(original_shape[2], tf.float32), tf.int32)
 new_time_size = tf.cast(tf.cast(original_shape[1], tf.float32)/(choosen_speed), tf.int32)
 spectrogram_aug = tf.image.resize(tf.expand_dims(spectrogram, -1), [new_time_size, new_freq_size])
 return spectrogram_aug[:, :, :, 0]
def augment_dropout(spectrogram,
keep_prob=0.9):
return tf.nn.dropout(spectrogram, rate=1-keep_prob)
 def transform_spectrogram(spectrogram,y):
spectrogram = tf.expand_dims(spectrogram, axis=0)
 if tf.random.uniform([])> 0.7:
spectrogram = augment_freq_time_mask(spectrogram)
 rand = tf.random.uniform([])
 if rand < 0.3:
spectrogram = augment_dropout(spectrogram, keep_prob=1-rand)
 if tf.random.uniform([])< 0.2:
spectrogram = augment_pitch_and_tempo(spectrogram)
 spectrogram = tf.squeeze(spectrogram, axis=0)
 return spectrogram,y
test_weights = np.array([1 / oof.shape[0]] * oof.shape[0] )<train_on_grid>
def paths_and_labels_to_dataset(audio_paths, labels, sec_labels):

 path_ds = tf.data.Dataset.from_tensor_slices(audio_paths)
 audio_ds = path_ds.map(lambda x: path_to_audio(x), num_parallel_calls=AUTO)
 label_ds = tf.data.Dataset.from_tensor_slices(to_categorical(labels))
sec_labels_ds = tf.data.Dataset.from_tensor_slices(sec_labels.apply(lambda x: sum(list(map(lambda x: to_categorical(x, NUM_CLASSES), x)))).tolist())
 label_ds = tf.data.Dataset.from_tensor_slices(sec_labels.apply(lambda x: sum(list(map(lambda x: to_categorical(x, NUM_CLASSES), x)))).tolist())
 return tf.data.Dataset.zip(( audio_ds, label_ds))
def path_to_audio(path):

 audio = tf.io.read_file(path)
 audio, sample_rate = tf.audio.decode_wav(audio, 1)
 return audio
def split_audio(audio, y):
len_audio = len(audio)
 effective_length = SAMPLE_RATE * PERIOD
if len_audio <= effective_length:
audio = tf.concat([audio, tf.expand_dims(tf.zeros([effective_length - len_audio]), 1)], axis=0)
 else:
start = tf.random.uniform([1], maxval = len_audio - effective_length, dtype = tf.int32)[0]
audio = audio[start:start + effective_length]
return audio, y
def audio_to_fft(audio, y):
audio = tf.squeeze(audio, axis=-1)
 fft = tf.signal.fft(
 tf.cast(tf.complex(real=audio, imag=tf.zeros_like(audio)) , tf.complex64)
)
 fft = tf.expand_dims(fft, axis=-1)
 return tf.math.abs(fft[:(audio.shape[0] // 2), :]), y
def audio_to_mel_spectrogram(audio, y):
audio = tf.squeeze(audio, axis=-1)
 spectrogram = tfio.experimental.audio.spectrogram(
 audio, nfft=2048, window=2048, stride=320)
 mel_spectrogram = tfio.experimental.audio.melscale(
 spectrogram, rate=SAMPLE_RATE, mels=500, fmin=50, fmax=14000)
 dbscale_mel_spectrogram = tfio.experimental.audio.dbscale(
 mel_spectrogram, top_db=80)
 return dbscale_mel_spectrogram, y
def mono_to_color(audio, y):
eps=1e-6
X = audio
X = tf.stack([X, X, X], axis=-1)
 mean = tf.math.reduce_mean(X)
 X = X - mean
std = tf.math.reduce_std(X)
 Xstd = X /(std + eps)
 _min, _max = tf.math.reduce_min(Xstd), tf.math.reduce_max(Xstd)
 norm_max = _max
norm_min = _min
if(_max - _min)> eps:
V = Xstd
V = 255 *(V - norm_min)/(norm_max - norm_min)
 else:
V = tf.zeros_like(Xstd)
 img_size = 224
image = tf.image.resize(V,(img_size,img_size))
return preprocess_input(image), y
tol = 1e-10
init_guess = [1 / oof.shape[0]] * oof.shape[0]
bnds = [(0, 1)for _ in range(oof.shape[0])]
cons = {'type': 'eq',
'fun': lambda x: np.sum(x)- 1,
'jac': lambda x: [1] * len(x)}
print('Inital Blend OOF:', func_numpy_metric(init_guess))
start_time = time()
res_scipy = minimize(fun = func_numpy_metric,
x0 = init_guess,
method = 'SLSQP',
jac = grad_func_jit,
bounds = bnds,
constraints = cons,
tol = tol)
 print(f'[{str(datetime.timedelta(seconds = time() - start_time)) [2:7]}] Optimised Blend OOF:', res_scipy.fun)
 print('Optimised Weights:', res_scipy.x )<normalization>
train_ds =(
 paths_and_labels_to_dataset(train_file_list['file_path'].values.tolist() , train_file_list['ebird_code'].values.tolist() , train_file_list['secondary_codes'])
.cache() 
.map(split_audio, num_parallel_calls=AUTO)
.map(audio_to_mel_spectrogram, num_parallel_calls=AUTO)
.map(transform_spectrogram, num_parallel_calls=AUTO)
.map(mono_to_color, num_parallel_calls=AUTO)
.shuffle(1024)
.repeat() 
.batch(BATCH_SIZE)
.map(functools.partial(mixup, BATCH_SIZE, 0.2), num_parallel_calls=AUTO)
.prefetch(AUTO)
)
 valid_ds =(
 paths_and_labels_to_dataset(val_file_list['file_path'].values.tolist() , val_file_list['ebird_code'].values.tolist() , val_file_list['secondary_codes'])
.map(split_audio, num_parallel_calls=AUTO)
.map(audio_to_mel_spectrogram, num_parallel_calls=AUTO)
.map(mono_to_color, num_parallel_calls=AUTO)
.cache() 
.batch(BATCH_SIZE)
.prefetch(AUTO)
 )
print('Check the sum of all weights:', np.sum(res_scipy.x))
if np.sum(res_scipy.x)- 1 <= tol:
print('Great! The sum of all weights equals to 1!')
 else:
print('Manual adjustion is needed to modify the weights.' )<load_from_csv>
class F1Callback(Callback):
def __init__(self):
self.f1s = []
def on_epoch_end(self, epoch, logs):
eps = np.finfo(np.float32 ).eps
recall = logs["val_clipwise_output_true_positives"] /(logs["val_clipwise_output_possible_positives"] + eps)
 precision = logs["val_clipwise_output_true_positives"] /(logs["val_clipwise_output_predicted_positives"] + eps)
 f1 = 2*precision*recall /(precision+recall+eps)
 print("f1_val(from log)=", f1)
 self.f1s.append(f1)
 def true_positives(y_true, y_pred):
return K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 def possible_positives(y_true, y_pred):
return K.sum(K.round(K.clip(y_true, 0, 1)))
 def predicted_positives(y_true, y_pred):
return K.sum(K.round(K.clip(y_pred, 0, 1)))
 def F1(y_true, y_pred):
TPFN = possible_positives(y_true, y_pred)
 TPFP = predicted_positives(y_true, y_pred)
 TP = true_positives(y_true, y_pred)
 return(TP * 2)/(TPFN + TPFP + K.epsilon())
 f1cb = F1Callback()
sub = pd.read_csv('/kaggle/input/lish-moa/sample_submission.csv')
 sub.iloc[:,1:] =(res_scipy.x[0]*sub_nn_transfer.iloc[:, 1:].values + \
res_scipy.x[1]*submission_TabNet.iloc[:,1:].values + \
res_scipy.x[2]*sub_rs.iloc[:,1:].values)
 sub.iloc[test_features['cp_type'] == 'ctl_vehicle',1:] = 0
sub.to_csv('submission.csv', index=False )<load_from_csv>
with strategy.scope() :
base_model = ResNet50(include_top=False, weights='imagenet')
 x = base_model.output
x = tf.reduce_mean(x, axis=2)
 x1 = L.MaxPooling1D(pool_size=3, strides=1, padding='same' )(x)
 x2 = L.AveragePooling1D(pool_size=3, strides=1, padding='same' )(x)
 x = x1 + x2
x = L.Dropout(0.5 )(x)
 x = L.Dense(1024, activation='relu' )(x)
 x = L.Dropout(0.5 )(x)
 norm_att = L.Conv1D(filters=NUM_CLASSES, kernel_size=1, padding='same' )(x)
 norm_att = tf.keras.activations.tanh(norm_att/10)*10
norm_att = tf.keras.activations.softmax(norm_att, axis=-2)
 segmentwise_output = L.Conv1D(filters=NUM_CLASSES, kernel_size=1, padding='same', activation='sigmoid', name='segmentwise_output' )(x)
 clipwise_output = tf.math.reduce_sum(norm_att * segmentwise_output, axis=1)
 clipwise_output = L.Lambda(lambda x: x, name="clipwise_output" )(clipwise_output)
 output = [segmentwise_output, clipwise_output]
model = tf.keras.models.Model(inputs=base_model.input, outputs=output)
 optimizer= tfa.optimizers.RectifiedAdam(
 lr=1e-3,
total_steps=10000,
warmup_proportion=0.1,
min_lr=1e-8, 
)
 model.compile(optimizer, loss=[None, "binary_crossentropy"],loss_weights=[0,1], metrics=[[],["accuracy", F1,true_positives,possible_positives,predicted_positives]])
 model.summary()
!rm submission_TabNet.csv<set_options>
%%time
es = tf.keras.callbacks.EarlyStopping(monitor='loss', verbose=1, patience=10)
 sv = tf.keras.callbacks.ModelCheckpoint('model.h5', monitor='val_loss', verbose=1, save_best_only=True, save_weights_only=True)
 reduce_lr = tf.keras.callbacks.ReduceLROnPlateau(monitor='loss', verbose=1, factor=0.5, patience=2, min_delta=0.0001, cooldown=1, min_lr=1e-7)
 STEPS_PER_EPOCH = len(train_file_list)// BATCH_SIZE
VALIDATION_STEP = len(val_file_list)// BATCH_SIZE
history = model.fit(
 train_ds,
epochs=100,
steps_per_epoch=STEPS_PER_EPOCH,
callbacks=[reduce_lr, es, sv, f1cb],
validation_data=valid_ds,
validation_steps = VALIDATION_STEP
)
<load_from_csv><EOS>