Kaggle competition: Rainforest Connection Species Audio Detection
<SOS> metric: WeightedLabelRankingAveragePrecision Kaggle data source: rainforest-connection-species-audio-detection<filter>
! pip install -q efficientnet
score_lgb.loc[score_lgb.fold == "oof"]<split>
import math, os, re, warnings, random
import tensorflow as tf
import numpy as np
import pandas as pd
import librosa
from kaggle_datasets import KaggleDatasets
import matplotlib.pyplot as plt
from IPython.display import Audio
from tensorflow.keras import Model, layers
from sklearn.model_selection import KFold
import tensorflow.keras.backend as K
from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, LearningRateScheduler
from tensorflow.keras.layers import GlobalAveragePooling2D, Input, Dense, Dropout, GaussianNoise
from tensorflow.keras.applications import ResNet50
import efficientnet.keras as efn
import seaborn as sns
oof_pred_xgb, test_pred_xgb, score_xgb, feat_imps_xgb = run_train_and_inference(
 X, X_test, y, "xgb", MODEL_PARAMS["xgb"], TRAIN_PARAMS["xgb"], RANDOM_SEED_LIST, N_SPLITS )<filter>
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print(f'Running on TPU {tpu.master() }')
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
AUTO = tf.data.experimental.AUTOTUNE
REPLICAS = strategy.num_replicas_in_sync
print(f'REPLICAS: {REPLICAS}' )
score_xgb.loc[score_xgb.fold == "oof"]<split>
def seed_everything(seed=0):
random.seed(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 os.environ['TF_DETERMINISTIC_OPS'] = '1'
seed = 42
seed_everything(seed)
 warnings.filterwarnings('ignore' )
oof_pred_cat, test_pred_cat, score_cat, feat_imps_cat = run_train_and_inference(
 X, X_test, y, "cat", MODEL_PARAMS["cat"], TRAIN_PARAMS["cat"], RANDOM_SEED_LIST, N_SPLITS )<filter>
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1)) for filename in filenames]
return np.sum(n)
 TRAIN_DATA_DIR = 'rfcx-audio-detection'
TRAIN_GCS_PATH = KaggleDatasets().get_gcs_path(TRAIN_DATA_DIR)
 FILENAMES = tf.io.gfile.glob(TRAIN_GCS_PATH + '/tp*.tfrec')
 TEST_DATA_DIR = 'rfcx-species-audio-detection'
TEST_GCS_PATH = KaggleDatasets().get_gcs_path(TEST_DATA_DIR)
 TEST_FILES = tf.io.gfile.glob(TEST_GCS_PATH + '/tfrecords/test/*.tfrec')
 no_of_training_samples = count_data_items(FILENAMES)
 print('num_training_samples are', no_of_training_samples )
score_cat.loc[score_cat.fold == "oof"]<define_variables>
CUT = 10
TIME = 10
EPOCHS = 25
GLOBAL_BATCH_SIZE = 4 * REPLICAS
LEARNING_RATE = 0.0015
WARMUP_LEARNING_RATE = 1e-5
WARMUP_EPOCHS = int(EPOCHS*0.1)
 PATIENCE = 8
STEPS_PER_EPOCH = 64
N_FOLDS = 5
NUM_TRAINING_SAMPLES = no_of_training_samples
class params:
sample_rate = 48000
stft_window_seconds: float = 0.025
stft_hop_seconds: float = 0.005
frame_length: int = 1200
mel_bands: int = 512
mel_min_hz: float = 50.0
mel_max_hz: float = 24000.0
log_offset: float = 0.001
patch_window_seconds: float = 0.96
patch_hop_seconds: float = 0.48
patch_frames = int(round(patch_window_seconds / stft_hop_seconds))
patch_bands = mel_bands
height = mel_bands
width = 2000
num_classes: int = 24
dropout = 0.35
classifier_activation: str = 'sigmoid'
model_names = ["lgb", "xgb", "cat"]<create_dataframe>
feature_description = {
'wav': tf.io.FixedLenFeature([], tf.string),
'recording_id': tf.io.FixedLenFeature([], tf.string),
'target' : tf.io.FixedLenFeature([], tf.float32),
'song_id': tf.io.FixedLenFeature([], tf.float32),
'tmin' : tf.io.FixedLenFeature([], tf.float32),
'fmin' : tf.io.FixedLenFeature([], tf.float32),
'tmax' : tf.io.FixedLenFeature([], tf.float32),
'fmax' : tf.io.FixedLenFeature([], tf.float32),
}
feature_dtype = {
'wav': tf.float32,
'recording_id': tf.string,
'target': tf.float32,
'song_id': tf.float32,
't_min': tf.float32,
'f_min': tf.float32,
't_max': tf.float32,
'f_max':tf.float32,
}
pd.DataFrame(
 np.corrcoef([
oof_pred_lgb,
oof_pred_xgb,
oof_pred_cat
]),
columns=model_names, index=model_names )<create_dataframe>
def waveform_to_log_mel_spectrogram(waveform,target_or_rec_id):

 window_length_samples = int(
 round(params.sample_rate * params.stft_window_seconds))
hop_length_samples = int(
 round(params.sample_rate * params.stft_hop_seconds))
fft_length = 2 ** int(np.ceil(np.log(window_length_samples)/ np.log(2.0)))
 num_spectrogram_bins = fft_length // 2 + 1
magnitude_spectrogram = tf.abs(tf.signal.stft(
 signals=waveform,
frame_length=params.frame_length,
frame_step=hop_length_samples,
fft_length= fft_length))
linear_to_mel_weight_matrix = tf.signal.linear_to_mel_weight_matrix(
 num_mel_bins=params.mel_bands,
num_spectrogram_bins=num_spectrogram_bins,
sample_rate=params.sample_rate,
lower_edge_hertz=params.mel_min_hz,
upper_edge_hertz=params.mel_max_hz)
 mel_spectrogram = tf.matmul(
 magnitude_spectrogram, linear_to_mel_weight_matrix)
 log_mel = tf.math.log(mel_spectrogram + params.log_offset)
 log_mel = tf.transpose(log_mel)
 log_mel_spectrogram = tf.reshape(log_mel , [tf.shape(log_mel)[0] ,tf.shape(log_mel)[1],1])
 spectrogram_hop_length_samples = int(
 round(params.sample_rate * params.stft_hop_seconds))
spectrogram_sample_rate = params.sample_rate / spectrogram_hop_length_samples
patch_window_length_samples = int(
 round(spectrogram_sample_rate * params.patch_window_seconds))
patch_hop_length_samples = int(
 round(spectrogram_sample_rate * params.patch_hop_seconds))
features = tf.signal.frame(
 signal=log_mel_spectrogram,
frame_length=patch_window_length_samples,
frame_step=patch_hop_length_samples,
axis=0)
 return log_mel_spectrogram, target_or_rec_id
pd.DataFrame(
 np.corrcoef([
test_pred_lgb,
test_pred_xgb,
test_pred_cat
]),
columns=model_names, index=model_names )<compute_test_metric>
def frequency_masking(mel_spectrogram):
frequency_masking_para = 80,
frequency_mask_num = 2
fbank_size = tf.shape(mel_spectrogram)
 n, v = fbank_size[0], fbank_size[1]
for i in range(frequency_mask_num):
f = tf.random.uniform([], minval=0, maxval= tf.squeeze(frequency_masking_para), dtype=tf.int32)
 v = tf.cast(v, dtype=tf.int32)
 f0 = tf.random.uniform([], minval=0, maxval= tf.squeeze(v-f), dtype=tf.int32)
 mask = tf.concat(( tf.ones(shape=(n, v - f0 - f,1)) ,
tf.zeros(shape=(n, f,1)) ,
tf.ones(shape=(n, f0,1)) , 
),1)
 mel_spectrogram = mel_spectrogram * mask
return tf.cast(mel_spectrogram, dtype=tf.float32)
 def time_masking(mel_spectrogram):
time_masking_para = 40,
time_mask_num = 1
fbank_size = tf.shape(mel_spectrogram)
 n, v = fbank_size[0], fbank_size[1]
for i in range(time_mask_num):
t = tf.random.uniform([], minval=0, maxval=tf.squeeze(time_masking_para), dtype=tf.int32)
 t0 = tf.random.uniform([], minval=0, maxval= n-t, dtype=tf.int32)
 mask = tf.concat(( tf.ones(shape=(n-t0-t, v,1)) ,
tf.zeros(shape=(t, v,1)) ,
tf.ones(shape=(t0, v,1)) , 
), 0)
 mel_spectrogram = mel_spectrogram * mask
return tf.cast(mel_spectrogram, dtype=tf.float32)
 def random_brightness(image):
return tf.image.random_brightness(image, 0.2)
 def random_gamma(image):
return tf.image.random_contrast(image, lower = 0.1, upper = 0.3)
 def random_flip_right(image):
return tf.image.random_flip_left_right(image)
 def random_flip_up_down(image):
return tf.image.random_flip_left_right(image)
 available_ops = [
frequency_masking ,
time_masking,
random_brightness,
random_flip_up_down,
random_flip_right
]
def apply_augmentation(image, target):
num_layers = int(np.random.uniform(low = 0, high = 3))
for layer_num in range(num_layers):
op_to_select = tf.random.uniform([], maxval=len(available_ops), dtype=tf.int32, seed = seed)
 for(i, op_name)in enumerate(available_ops):
image = tf.cond(
 tf.equal(i, op_to_select),
lambda selected_func=op_name,: selected_func(
 image),
lambda: image)
 return image, target
oof_pred_avg =(oof_pred_lgb + oof_pred_xgb + oof_pred_cat)/ 3
oof_score_avg = mean_squared_error(y, oof_pred_avg, squared=False)
 print(f"oof score avg: {oof_score_avg:.5f}" )<compute_test_metric>
def preprocess(image, target_or_rec_id):
image = tf.image.grayscale_to_rgb(image)
 image = tf.image.resize(image, [params.height,params.width])
 image = tf.image.per_image_standardization(image)
 return image , target_or_rec_id
def read_labeled_tfrecord(example_proto):
sample = tf.io.parse_single_example(example_proto, feature_description)
 wav, _ = tf.audio.decode_wav(sample['wav'], desired_channels=1)
 target = tf.cast(sample['target'],tf.float32)
 target = tf.squeeze(tf.one_hot([target,], depth = params.num_classes), axis = 0)
 tmin = tf.cast(sample['tmin'], tf.float32)
 fmin = tf.cast(sample['fmin'], tf.float32)
 tmax = tf.cast(sample['tmax'], tf.float32)
 fmax = tf.cast(sample['fmax'], tf.float32)
 tmax_s = tmax * tf.cast(params.sample_rate, tf.float32)
 tmin_s = tmin * tf.cast(params.sample_rate, tf.float32)
 cut_s = tf.cast(CUT * params.sample_rate, tf.float32)
 all_s = tf.cast(60 * params.sample_rate, tf.float32)
 tsize_s = tmax_s - tmin_s
cut_min = tf.cast(
 tf.maximum(0.0,
tf.minimum(tmin_s -(cut_s - tsize_s)/ 2,
tf.minimum(tmax_s +(cut_s - tsize_s)/ 2, all_s)- cut_s)
), tf.int32 
)
 cut_max = cut_min + CUT * params.sample_rate
wav = tf.squeeze(wav[cut_min : cut_max])
 return wav, target
def read_unlabeled_tfrecord(example):
feature_description = {
'recording_id': tf.io.FixedLenFeature([], tf.string),
'audio_wav': tf.io.FixedLenFeature([], tf.string),
}
sample = tf.io.parse_single_example(example, feature_description)
 wav, _ = tf.audio.decode_wav(sample['audio_wav'], desired_channels=1)
 recording_id = tf.reshape(tf.cast(sample['recording_id'] , tf.string), [1])
 def _cut_audio(i):
_sample = {
'audio_wav': tf.reshape(wav[i*params.sample_rate*TIME:(i+1)*params.sample_rate*TIME], [params.sample_rate*TIME]),
'recording_id': sample['recording_id']
}
return _sample
return tf.map_fn(_cut_audio, tf.range(60//TIME), dtype={
'audio_wav': tf.float32,
'recording_id': tf.string
} )
test_pred_avg =(test_pred_lgb + test_pred_xgb + test_pred_cat)/ 3<save_to_csv>
def load_dataset(filenames, labeled = True, ordered = False , training = True):
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads = AUTO)
 dataset = dataset.map(read_labeled_tfrecord , num_parallel_calls = AUTO)
 dataset = dataset.map(waveform_to_log_mel_spectrogram , num_parallel_calls = AUTO)
 if training:
dataset = dataset.map(apply_augmentation, num_parallel_calls = AUTO)
 dataset = dataset.map(preprocess, num_parallel_calls = AUTO)
 return dataset
sub = smpl_sub.copy()
sub[TGT_COL] = test_pred_avg
sub.to_csv("submission.csv", index=False)
 sub.head()<set_options>
def get_dataset(filenames, training = True):
if training:
dataset = load_dataset(filenames , training = True)
 dataset = dataset.shuffle(256 ).repeat()
dataset = dataset.batch(GLOBAL_BATCH_SIZE, drop_remainder = True)
 else:
dataset = load_dataset(filenames , training = False)
 dataset = dataset.batch(GLOBAL_BATCH_SIZE ).cache()
dataset = dataset.prefetch(AUTO)
 return dataset
InteractiveShell.ast_node_interactivity = "all"
%matplotlib inline<load_from_csv>
def _one_sample_positive_class_precisions(example):
y_true, y_pred = example
y_true = tf.reshape(y_true, tf.shape(y_pred))
retrieved_classes = tf.argsort(y_pred, direction='DESCENDING')
 class_rankings = tf.argsort(retrieved_classes)
 retrieved_class_true = tf.gather(y_true, retrieved_classes)
 retrieved_cumulative_hits = tf.math.cumsum(tf.cast(retrieved_class_true, tf.float32))
idx = tf.where(y_true)[:, 0]
i = tf.boolean_mask(class_rankings, y_true)
 r = tf.gather(retrieved_cumulative_hits, i)
 c = 1 + tf.cast(i, tf.float32)
 precisions = r / c
dense = tf.scatter_nd(idx[:, None], precisions, [y_pred.shape[0]])
 return dense
class LWLRAP(tf.keras.metrics.Metric):
def __init__(self, num_classes, name='lwlrap'):
super().__init__(name=name)
 self._precisions = self.add_weight(
 name='per_class_cumulative_precision',
shape=[num_classes],
initializer='zeros', 
)
 self._counts = self.add_weight(
 name='per_class_cumulative_count',
shape=[num_classes],
initializer='zeros', 
)
 def update_state(self, y_true, y_pred, sample_weight=None):
precisions = tf.map_fn(
 fn=_one_sample_positive_class_precisions,
elems=(y_true, y_pred),
dtype=(tf.float32), 
)
 increments = tf.cast(precisions > 0, tf.float32)
 total_increments = tf.reduce_sum(increments, axis=0)
 total_precisions = tf.reduce_sum(precisions, axis=0)
 self._precisions.assign_add(total_precisions)
 self._counts.assign_add(total_increments)
 def result(self):
per_class_lwlrap = self._precisions / tf.maximum(self._counts, 1.0)
 per_class_weight = self._counts / tf.reduce_sum(self._counts)
 overall_lwlrap = tf.reduce_sum(per_class_lwlrap * per_class_weight)
 return overall_lwlrap
def reset_states(self):
self._precisions.assign(self._precisions * 0)
 self._counts.assign(self._counts * 0 )
train_df = pd.read_csv('.. /input/tabular-playground-series-jan-2021/train.csv')
 test_df = pd.read_csv('.. /input/tabular-playground-series-jan-2021/test.csv')
 sub_df = pd.read_csv('.. /input/tabular-playground-series-jan-2021/sample_submission.csv')
 seed = 66<count_missing_values>
def cosine_decay_with_warmup(global_step,
learning_rate_base,
total_steps,
warmup_learning_rate=0.0,
warmup_steps= 0,
hold_base_rate_steps=0):
if total_steps < warmup_steps:
raise ValueError('total_steps must be larger or equal to '
'warmup_steps.')
 learning_rate = 0.5 * learning_rate_base *(1 + tf.cos(
 np.pi * 
(tf.cast(global_step, tf.float32)- warmup_steps - hold_base_rate_steps 
)/ float(total_steps - warmup_steps - hold_base_rate_steps)))
 if hold_base_rate_steps > 0:
learning_rate = tf.where(
 global_step > warmup_steps + hold_base_rate_steps,
learning_rate, learning_rate_base)
 if warmup_steps > 0:
if learning_rate_base < warmup_learning_rate:
raise ValueError('learning_rate_base must be larger or equal to '
'warmup_learning_rate.')
 slope =(learning_rate_base - warmup_learning_rate)/ warmup_steps
warmup_rate = slope * tf.cast(global_step,
tf.float32)+ warmup_learning_rate
learning_rate = tf.where(global_step < warmup_steps, warmup_rate,
learning_rate)
 return tf.where(global_step > total_steps, 0.0, learning_rate,
name='learning_rate')
 rng = [i for i in range(int(EPOCHS * STEPS_PER_EPOCH)) ]
WARMUP_STEPS = int(WARMUP_EPOCHS * STEPS_PER_EPOCH)
 y = [cosine_decay_with_warmup(x , LEARNING_RATE, len(rng), 1e-5, WARMUP_STEPS)for x in rng]
sns.set(style='whitegrid')
 fig, ax = plt.subplots(figsize=(20, 6))
plt.plot(rng, y )
train_df.isna().sum().sum()
test_df.isna().sum().sum()<train_model>
class WarmUpCosineDecayScheduler(tf.keras.callbacks.Callback):
def __init__(self,
learning_rate_base,
total_steps,
global_step_init=0,
warmup_learning_rate=0.0,
warmup_steps=0,
hold_base_rate_steps=0,
verbose=0):
super(WarmUpCosineDecayScheduler, self ).__init__()
self.learning_rate_base = learning_rate_base
self.total_steps = total_steps
self.global_step = global_step_init
self.warmup_learning_rate = warmup_learning_rate
self.warmup_steps = warmup_steps
self.hold_base_rate_steps = hold_base_rate_steps
self.verbose = verbose
self.learning_rates = []
def on_batch_end(self, batch, logs=None):
self.global_step = self.global_step + 1
lr = K.get_value(self.model.optimizer.lr)
 self.learning_rates.append(lr)
 def on_batch_begin(self, batch, logs=None):
lr = cosine_decay_with_warmup(global_step=self.global_step,
learning_rate_base=self.learning_rate_base,
total_steps=self.total_steps,
warmup_learning_rate=self.warmup_learning_rate,
warmup_steps=self.warmup_steps,
hold_base_rate_steps=self.hold_base_rate_steps)
 K.set_value(self.model.optimizer.lr, lr)
 if self.verbose > 0:
print('
Batch %05d: setting learning '
'rate to %s.' %(self.global_step + 1, lr.numpy()))
total_steps = int(EPOCHS * STEPS_PER_EPOCH)
 warmup_steps = int(WARMUP_EPOCHS * STEPS_PER_EPOCH)
 warmup_learning_rate = WARMUP_LEARNING_RATE
X = train_df.drop(['id', 'target'], axis=1)
 y = train_df.target
estim = DecisionTreeRegressor()
estim.fit(X,y )<compute_test_metric>
def RFCX_MODEL() :
waveform = Input(shape=(None,None,3), dtype=tf.float32)
 noisy_waveform = GaussianNoise(0.2 )(waveform)
 model = efn.EfficientNetB2(include_top=False, weights='imagenet',)
 model_output = model(noisy_waveform)
 model_output = GlobalAveragePooling2D()(model_output)
 dense = Dropout(params.dropout )(model_output)
 predictions = Dense(params.num_classes, activation = params.classifier_activation )(dense)
 model = Model(
 name='Efficientnet', inputs=waveform,
outputs=[predictions])
 return model
X, y<compute_test_metric>
def get_model() :
with strategy.scope() :
model = RFCX_MODEL()
model.summary()
model.compile(optimizer = 'adam',
loss = tf.keras.losses.BinaryCrossentropy(label_smoothing = 0.1),
metrics = [LWLRAP(num_classes = params.num_classes),
])
 return model
F_test = f_regression(X, y)
 F_test<split>
skf = KFold(n_splits=N_FOLDS, shuffle=True, random_state=seed)
 oof_pred = []; oof_labels = []; history_list = []
for fold,(idxT, idxV)in enumerate(skf.split(np.arange(10))):
if tpu: tf.tpu.experimental.initialize_tpu_system(tpu)
 print(f'
FOLD: {fold+1}')
 print(f'TRAIN: {idxT} VALID: {idxV}')
 TRAIN_FILENAMES = [FILENAMES[x] for x in idxT]
VALID_FILENAMES = [FILENAMES[x] for x in idxV]
np.random.shuffle(TRAIN_FILENAMES)
 train_dataset = get_dataset(TRAIN_FILENAMES, training=True,)
 validation_data= get_dataset(VALID_FILENAMES, training=False)
 model = get_model()
model_path = f'RFCX_model_fold {fold}.h5'
early_stopping = EarlyStopping(monitor = 'val_lwlrap', mode = 'max',
patience = PATIENCE, restore_best_weights=True, verbose=1)
 cosine_warm_up_lr = WarmUpCosineDecayScheduler(learning_rate_base= LEARNING_RATE,
total_steps= total_steps,
warmup_learning_rate= warmup_learning_rate,
warmup_steps= warmup_steps,
hold_base_rate_steps=0)
 history = model.fit(train_dataset,
steps_per_epoch=STEPS_PER_EPOCH,
callbacks=[early_stopping, cosine_warm_up_lr],
epochs=EPOCHS,
validation_data = validation_data,
verbose = 2 ).history
history_list.append(history)
 model.save_weights(model_path)
 ds_valid = get_dataset(VALID_FILENAMES, training = False)
 oof_labels.append([target.numpy() for frame, target in iter(ds_valid.unbatch())])
 x_oof = ds_valid.map(lambda frames, target: frames)
 oof_pred.append(np.argmax(model.predict(x_oof), axis=-1))
print(f"
X_train, X_val, y_train, y_val = train_test_split(
 X, y, test_size=0.2, random_state=seed)
 def opt_lgm(trial, rounds=10):
param = {
'boosting_type': trial.suggest_categorical('boosting_type', ['gbdt', 'rf', 'dart']),
'lambda_l1': trial.suggest_loguniform('lambda_l1', 1e-8, 10.0),
'lambda_l2': trial.suggest_loguniform('lambda_l2', 1e-8, 10.0),
'feature_fraction': trial.suggest_uniform('feature_fraction', 0.4, 1.0),
'bagging_fraction': trial.suggest_uniform('bagging_fraction', 0.4, 1.0),
'bagging_freq': trial.suggest_int('bagging_freq', 1, 7),
'min_child_samples': trial.suggest_int('min_child_samples', 5, 100),
'learning_rate': trial.suggest_uniform('learning_rate', 0.005, 0.1),
'importance_type': ['gain', 'split']
}
param['n_estimators'] = 500
model = lgb.LGBMRegressor(**param)
 model = model.fit(X_train, y_train)
 valid_prediction = model.predict(X_val)
 mse = mean_squared_error(y_val, valid_prediction, squared=False)
 return mse
study = optuna.create_study(direction='minimize')
 study.optimize(opt_lgm, n_trials=2)
 params = study.best_params<predict_on_test>
def get_test_dataset(filenames, training = False):
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads = AUTO)
 dataset = dataset.map(read_unlabeled_tfrecord , num_parallel_calls = AUTO ).unbatch()
dataset = dataset.map(lambda spec : waveform_to_log_mel_spectrogram(spec['audio_wav'], spec['recording_id']), num_parallel_calls = AUTO)
 dataset = dataset.map(preprocess, num_parallel_calls = AUTO)
 return dataset.batch(GLOBAL_BATCH_SIZE*4 ).cache()
params['n_estimators'] = 500
model = lgb.LGBMRegressor(**params)
 kf = KFold(n_splits=5)
 i=0
for train_index, test_index in kf.split(X):
X_train, X_test = X.loc[train_index, :], X.loc[test_index, :]
y_train, y_test = y[train_index], y[test_index]
clf = model.fit(X_train, y_train)
 preds = clf.predict(X_test)
 print('mse:', mean_squared_error(y_test, preds, squared=False))
sub_preds = test_df.drop(['id'], axis=1)
 sub_df[f'pred_{i}'] = clf.predict(sub_preds)
 i+=1<feature_engineering>
test_predict = []
test_data = get_test_dataset(TEST_FILES, training = False)
 test_audio = test_data.map(lambda frames, recording_id: frames)
 for fold in range(N_FOLDS):
model.load_weights(f'./RFCX_model_fold {fold}.h5')
 test_predict.append(model.predict(test_audio, verbose = 1))
sub_df['target'] = sub_df.drop(['id', 'target'], axis=1 ).mean(axis=1)
 sub_df['target'] = sub_df['target']
sub_df = sub_df[['id', 'target']]
sub_df.head(3 )<save_to_csv>
SUB = pd.read_csv('.. /input/rfcx-species-audio-detection/sample_submission.csv')
 predict = np.array(test_predict ).reshape(N_FOLDS, len(SUB), 60 // TIME, params.num_classes)
 predict = np.mean(np.max(predict ,axis = 2), axis = 0)
 recording_id = test_data.map(lambda frames, recording_id: recording_id ).unbatch()
test_ids = next(iter(recording_id.batch(len(SUB)* 60 // TIME)) ).numpy().astype('U' ).reshape(len(SUB), 60 // TIME)
 pred_df = pd.DataFrame({ 'recording_id' : test_ids[:, 0],
**{f's{i}' : predict[:, i] for i in range(params.num_classes)} } )
<set_options><EOS> 
pred_df.sort_values('recording_id', inplace = True)
 pred_df.to_csv('submission.csv', index = False )