Kaggle competition: Rainforest Connection Species Audio Detection
<SOS> metric: WeightedLabelRankingAveragePrecision Kaggle data source: rainforest-connection-species-audio-detection<save_to_csv>
COLAB=False
models_path=''
if not os.path.exists('.. /input/rfcx-species-audio-detection'):
COLAB=True
drive.mount('/content/drive')
 GCS_DS_PATH = 'gs://kds-5c677f76ce55440722b2a474a5492faa70847c05a8f5d722f5a37feb'
models_path='/content/drive/MyDrive/Models/'
else:
GCS_DS_PATH = KaggleDatasets().get_gcs_path('rfcx-species-audio-detection')
 print(GCS_DS_PATH )
concat_sub['isFraud'] = concat_sub['ieee_mean']
concat_sub[['TransactionID', 'isFraud']].to_csv('stack_mean.csv',
index=False, float_format='%.6f' )<save_to_csv>
if COLAB:
if not os.path.exists('/.kaggle/kaggle.json'):
!mkdir ~/.kaggle
if not os.path.exists('/content/drive/My Drive/Models/kaggle.json'):
files.upload()
!cp kaggle.json ~/.kaggle/
else:
!cp '/content/drive/My Drive/Models/kaggle.json' ~/.kaggle/
!chmod 600 ~/.kaggle/kaggle.json
concat_sub['isFraud'] = concat_sub['ieee_median']
concat_sub[['TransactionID', 'isFraud']].to_csv('stack_median.csv',
index=False, float_format='%.6f' )<save_to_csv>
if COLAB:
!pip install -q tensorflow~=2.2.0 tensorflow_gcs_config~=2.2.0
resp = requests.post("http://{}:8475/requestversion/{}".format(os.environ["COLAB_TPU_ADDR"].split(":")[0], tf.__version__))
if resp.status_code != 200:
print("Failed to switch the TPU to TF {}".format(version))
concat_sub['isFraud'] = np.where(np.all(concat_sub.iloc[:,1:ncol] > cutoff_lo, axis=1), 1,
np.where(np.all(concat_sub.iloc[:,1:ncol] < cutoff_hi, axis=1),
0, concat_sub['ieee_median']))
concat_sub[['TransactionID', 'isFraud']].to_csv('stack_pushout_median.csv',
index=False, float_format='%.6f' )<feature_engineering>
if COLAB:
print("Tensorflow version " + tf.__version__)
 try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.cluster_spec().as_dict() ['worker'])
 except ValueError:
raise BaseException('ERROR: Not connected to a TPU runtime; please see the previous cell in this notebook for instructions!')
 tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 tpu_strategy = tf.distribute.experimental.TPUStrategy(tpu)
concat_sub['isFraud'] = np.where(np.all(concat_sub.iloc[:,1:ncol] > cutoff_lo, axis=1),
concat_sub['ieee_max'],
np.where(np.all(concat_sub.iloc[:,1:ncol] < cutoff_hi, axis=1),
concat_sub['ieee_min'],
concat_sub['ieee_mean']))
concat_sub[['TransactionID', 'isFraud']].to_csv('stack_minmax_mean.csv',
index=False, float_format='%.6f' )<feature_engineering>
!pip install image-classifiers
!pip install tensorflow_addons==0.10.0
!pip install soundfile
tf.__version__
concat_sub['isFraud'] = np.where(np.all(concat_sub.iloc[:,1:ncol] > cutoff_lo, axis=1),
concat_sub['ieee_max'],
np.where(np.all(concat_sub.iloc[:,1:ncol] < cutoff_hi, axis=1),
concat_sub['ieee_min'],
concat_sub['ieee_median']))
concat_sub[['TransactionID', 'isFraud']].to_csv('stack_minmax_median.csv',
index=False, float_format='%.6f' )<load_from_csv>
from classification_models.keras import Classifiers
sub_base = pd.read_csv('.. /input/ieeesubmissions/blend01.csv' )<feature_engineering>
SEED = random.randint(0, 10000)
 def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 seed_everything(SEED )
concat_sub['ieee_base'] = sub_base['isFraud']
concat_sub['isFraud'] = np.where(np.all(concat_sub.iloc[:,1:ncol] > cutoff_lo, axis=1),
concat_sub['ieee_max'],
np.where(np.all(concat_sub.iloc[:,1:ncol] < cutoff_hi, axis=1),
concat_sub['ieee_min'],
concat_sub['ieee_base']))
concat_sub[['TransactionID', 'isFraud']].to_csv('stack_minmax_bestbase.csv',
index=False, float_format='%.6f' )<set_options>
ResNet34, preprocess_input = Classifiers.get('resnet34' )
gc.enable()
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
 %matplotlib inline
py.init_notebook_mode(connected=True)
 warnings.filterwarnings('ignore')
 GradientBoostingClassifier, ExtraTreesClassifier)
<define_variables>
cfg = {
'parse_params': {
'cut_time': 10,
},
'data_params': {
'sample_time': 6,
'spec_fmax': 24000.0,
'spec_fmin': 40.0,
'spec_mel': 300,
'mel_power': 2,
'img_shape':(300, 670)
 },
'model_params': {
'batchsize_per_tpu': 8,
'iteration_per_epoch': 128,
'epoch': 25,
'arch': ResNet34,
'arch_preprocess': preprocess_input,
'freeze_to': 0,
'loss': {
'fn': tfa.losses.SigmoidFocalCrossEntropy,
'params': {},
},
'optim': {
'fn': tfa.optimizers.RectifiedAdam,
'params': {'lr': 2e-3, 'total_steps': 18*64, 'warmup_proportion': 0.3, 'min_lr': 1e-6},
},
'mixup': True
}
}
useXGB = False
useLGBM = True
useEnsemble = False
if(useXGB)and(useLGBM):
useEnsemble = False<load_from_csv>
if not COLAB:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 print("All devices: ", tf.config.list_logical_devices('TPU'))
tpu_strategy = tf.distribute.experimental.TPUStrategy(tpu )
%%time
train_transaction = pd.read_csv('.. /input/train_transaction.csv', index_col='TransactionID')
 test_transaction = pd.read_csv('.. /input/test_transaction.csv', index_col='TransactionID')
 train_identity = pd.read_csv('.. /input/train_identity.csv', index_col='TransactionID')
 test_identity = pd.read_csv('.. /input/test_identity.csv', index_col='TransactionID')
 sample_submission = pd.read_csv('.. /input/sample_submission.csv', index_col='TransactionID')
 train = train_transaction.merge(train_identity, how='left', left_index=True, right_index=True)
 test = test_transaction.merge(test_identity, how='left', left_index=True, right_index=True)
 print(train.shape)
 print(test.shape )<feature_engineering>
AUTOTUNE = tf.data.experimental.AUTOTUNE
TRAIN_TFREC = GCS_DS_PATH + "/tfrecords/train"
TEST_TFREC = GCS_DS_PATH + "/tfrecords/test"
def make_day_feature(df, offset=0, tname='TransactionDT'):

 days = df[tname] /(3600*24)
 encoded_days = np.floor(days-1+offset)% 7
return encoded_days
def make_hour_feature(df, tname='TransactionDT'):

 hours = df[tname] /(3600)
 encoded_hours = np.floor(hours)% 24
return encoded_hours 
<feature_engineering>
CUT = cfg['parse_params']['cut_time']
SR = 48000
TIME = cfg['data_params']['sample_time']
FMAX = cfg['data_params']['spec_fmax']
FMIN = cfg['data_params']['spec_fmin']
N_MEL = cfg['data_params']['spec_mel']
HEIGHT, WIDTH = cfg['data_params']['img_shape']
CLASS_N = 24
train['hours'] = make_hour_feature(train)
 test['hours']= make_hour_feature(test )<categorify>
raw_dataset = tf.data.TFRecordDataset([TRAIN_TFREC + '/00-148.tfrec'])
 raw_dataset
for f in train.columns:
if train[f].dtype=='object' or train[f].dtype=='object':
lbl = preprocessing.LabelEncoder()
lbl.fit(list(train[f].values)+ list(test[f].values))
train[f] = lbl.transform(list(train[f].values))
test[f] = lbl.transform(list(test[f].values))<prepare_x_and_y>
feature_description = {
'recording_id': tf.io.FixedLenFeature([], tf.string, default_value=''),
'audio_wav': tf.io.FixedLenFeature([], tf.string, default_value=''),
'label_info': tf.io.FixedLenFeature([], tf.string, default_value=''),
}
parse_dtype = {
'audio_wav': tf.float32,
'recording_id': tf.string,
'species_id': tf.int32,
'songtype_id': tf.int32,
't_min': tf.float32,
'f_min': tf.float32,
't_max': tf.float32,
'f_max':tf.float32,
'is_tp': tf.int32
}
@tf.function
def _parse_function(example_proto):
sample = tf.io.parse_single_example(example_proto, feature_description)
 wav, _ = tf.audio.decode_wav(sample['audio_wav'], desired_channels=1)
 label_info = tf.strings.split(sample['label_info'], sep='"')[1]
labels = tf.strings.split(label_info, sep=';')
 @tf.function
def _cut_audio(label):
items = tf.strings.split(label, sep=',')
 spid = tf.squeeze(tf.strings.to_number(items[0], tf.int32))
soid = tf.squeeze(tf.strings.to_number(items[1], tf.int32))
tmin = tf.squeeze(tf.strings.to_number(items[2]))
fmin = tf.squeeze(tf.strings.to_number(items[3]))
tmax = tf.squeeze(tf.strings.to_number(items[4]))
fmax = tf.squeeze(tf.strings.to_number(items[5]))
tp = tf.squeeze(tf.strings.to_number(items[6], tf.int32))
tmax_s = tmax * tf.cast(SR, tf.float32)
 tmin_s = tmin * tf.cast(SR, tf.float32)
 cut_s = tf.cast(CUT * SR, tf.float32)
 all_s = tf.cast(60 * SR, tf.float32)
 tsize_s = tmax_s - tmin_s
cut_min = tf.cast(
 tf.maximum(0.0,
tf.minimum(tmin_s -(cut_s - tsize_s)/ 2,
tf.minimum(tmax_s +(cut_s - tsize_s)/ 2, all_s)- cut_s)
), tf.int32 
)
 cut_max = cut_min + CUT * SR
_sample = {
'audio_wav': tf.reshape(wav[cut_min:cut_max], [CUT*SR]),
'recording_id': sample['recording_id'],
'species_id': spid,
'songtype_id': soid,
't_min': tmin - tf.cast(cut_min, tf.float32)/tf.cast(SR, tf.float32),
'f_min': fmin,
't_max': tmax - tf.cast(cut_min, tf.float32)/tf.cast(SR, tf.float32),
'f_max': fmax,
'is_tp': tp
}
return _sample
samples = tf.map_fn(_cut_audio, labels, dtype=parse_dtype)
 return samples
parsed_dataset = raw_dataset.map(_parse_function ).unbatch()
y_train = train['isFraud'].copy()
del train_transaction, train_identity, test_transaction, test_identity
X_train = train.drop('isFraud', axis=1)
 X_test = test.copy()
del train, test<drop_column>
@tf.function
def _cut_wav(x):
cut_min = tf.random.uniform([], maxval=(CUT-TIME)*SR, dtype=tf.int32)
 cut_max = cut_min + TIME * SR
cutwave = tf.reshape(x['audio_wav'][cut_min:cut_max], [TIME*SR])
 y = {}
y.update(x)
 y['audio_wav'] = cutwave
y['t_min'] = tf.maximum(0.0, x['t_min'] - tf.cast(cut_min, tf.float32)/ SR)
 y['t_max'] = tf.maximum(0.0, x['t_max'] - tf.cast(cut_min, tf.float32)/ SR)
 return y
@tf.function
def _cut_wav_val(x):
cut_min =(CUT-TIME)*SR // 2
cut_max = cut_min + TIME * SR
cutwave = tf.reshape(x['audio_wav'][cut_min:cut_max], [TIME*SR])
 y = {}
y.update(x)
 y['audio_wav'] = cutwave
y['t_min'] = tf.maximum(0.0, x['t_min'] - tf.cast(cut_min, tf.float32)/ SR)
 y['t_max'] = tf.maximum(0.0, x['t_max'] - tf.cast(cut_min, tf.float32)/ SR)
 return y
drop_col = dtcol +mcol[0:len(mcol)]+ idcol[5:39] + dcol[0:len(dcol)] + ccol[4:len(ccol)]
X_train.drop(drop_col,axis=1, inplace=True)
 X_test.drop(drop_col, axis=1, inplace=True)
 X_train.head()<feature_engineering>
@tf.function
def _filtTP(x):
return x['is_tp'] == 1
def make_amt_feature(df, tname='TransactionAmt'):

 logAmt = np.log(df[tname])
 return logAmt<feature_engineering>
def show_wav(sample, ax):
wav = sample["audio_wav"].numpy()
rate = SR
ax.plot(np.arange(len(wav)) / rate, wav)
 ax.set_title(
 sample["recording_id"].numpy().decode()
+("/%d" % sample["species_id"])
 +("TP" if sample["is_tp"] else "FP"))
return Audio(( wav * 2**15 ).astype(np.int16), rate=rate)
 fig, ax = plt.subplots(figsize=(15, 3))
show_wav(next(iter(parsed_dataset)) , ax )
X_train['LogAmt'] = make_amt_feature(X_train)
 X_test['LogAmt']= make_amt_feature(X_test )<train_model>
@tf.function
def _wav_to_spec(x):
mel_power = cfg['data_params']['mel_power']
stfts = tf.signal.stft(x["audio_wav"], frame_length=2048, frame_step=512, fft_length=2048)
 spectrograms = tf.abs(stfts)** mel_power
num_spectrogram_bins = stfts.shape[-1]
lower_edge_hertz, upper_edge_hertz, num_mel_bins = FMIN, FMAX, N_MEL
linear_to_mel_weight_matrix = tf.signal.linear_to_mel_weight_matrix(
 num_mel_bins, num_spectrogram_bins, SR, lower_edge_hertz,
upper_edge_hertz)
 mel_spectrograms = tf.tensordot(
 spectrograms, linear_to_mel_weight_matrix, 1)
 mel_spectrograms.set_shape(spectrograms.shape[:-1].concatenate(
 linear_to_mel_weight_matrix.shape[-1:]))
log_mel_spectrograms = tf.math.log(mel_spectrograms + 1e-6)
 y = {
'audio_spec': tf.transpose(log_mel_spectrograms),
}
y.update(x)
 return y
spec_dataset = parsed_dataset.filter(_filtTP ).map(_cut_wav ).map(_wav_to_spec )
%%time
if useXGB :
NFOLDS = 6
kf = StratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=123)
 y_preds_xgb = np.zeros(X_test.shape[0])
 y_oof_xgb = np.zeros(X_train.shape[0])
 score = 0
for fold,(tr_idx, val_idx)in enumerate(kf.split(X_train, y_train)) :
clf = xgb.XGBClassifier(
 n_estimators=500,
max_depth=9,
learning_rate=0.05,
subsample=0.9,
colsample_bytree=0.9,
gamma = 0.2,
alpha = 4,
missing = -1,
tree_method='gpu_hist' 
)
 X_tr, X_vl = X_train.iloc[tr_idx, :], X_train.iloc[val_idx, :]
y_tr, y_vl = y_train.iloc[tr_idx], y_train.iloc[val_idx]
clf.fit(X_tr, y_tr)
 y_pred_train = clf.predict_proba(X_vl)[:,1]
y_oof_xgb[val_idx] = y_pred_train
print("FOLD: ",fold,' AUC {}'.format(roc_auc_score(y_vl, y_pred_train)))
 score += roc_auc_score(y_vl, y_pred_train)/ NFOLDS
y_preds_xgb+= clf.predict_proba(X_test)[:,1] / NFOLDS
del X_tr, X_vl, y_tr, y_vl
gc.collect()
y_preds_xgb.reshape(-1,1)
 y_oof_xgb.reshape(-1,1)
 print("
MEAN AUC = {}".format(score))
print("OOF AUC = {}".format(roc_auc_score(y_train, y_oof_xgb)) )<split>
@tf.function
def _create_annot(x):
targ = tf.one_hot(x["species_id"], CLASS_N, on_value=x["is_tp"], off_value=0)
 return {
'input': x["audio_spec"],
'target': tf.cast(targ, tf.float32)
 }
annot_dataset = spec_dataset.map(_create_annot )
%%time
if useXGB :
NFOLDS = 5
kf = StratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=0)
 y_preds_xgb1 = np.zeros(X_test.shape[0])
 y_oof_xgb1 = np.zeros(X_train.shape[0])
 score = 0
for fold,(tr_idx, val_idx)in enumerate(kf.split(X_train, y_train)) :
clf = xgb.XGBClassifier(
 n_estimators=1500,
max_depth=10,
learning_rate=0.05,
subsample=0.9,
colsample_bytree=0.9,
gamma = 0.2,
alpha = 4,
missing = -1,
tree_method='gpu_hist' 
)
 X_tr, X_vl = X_train.iloc[tr_idx, :], X_train.iloc[val_idx, :]
y_tr, y_vl = y_train.iloc[tr_idx], y_train.iloc[val_idx]
clf.fit(X_tr, y_tr)
 y_pred_train = clf.predict_proba(X_vl)[:,1]
y_oof_xgb1[val_idx] = y_pred_train
print("FOLD: ",fold,' AUC {}'.format(roc_auc_score(y_vl, y_pred_train)))
 score += roc_auc_score(y_vl, y_pred_train)/ NFOLDS
y_preds_xgb1+= clf.predict_proba(X_test)[:,1] / NFOLDS
del X_tr, X_vl, y_tr, y_vl
gc.collect()
y_preds_xgb1.reshape(-1,1)
 y_oof_xgb1.reshape(-1,1)
 print("
MEAN AUC = {}".format(score))
print("OOF AUC = {}".format(roc_auc_score(y_train, y_oof_xgb1)) )<split>
def _preprocess_img(x, training=False, test=False):
image = tf.expand_dims(x, axis=-1)
 image = tf.image.resize(image, [HEIGHT, WIDTH])
 image = tf.image.per_image_standardization(image)
 @tf.function
def _specaugment(image):
ERASE_TIME = 50
ERASE_MEL = 16
image = tf.expand_dims(image, axis=0)
 xoff = tf.random.uniform([2], minval=ERASE_TIME//2, maxval=WIDTH-ERASE_TIME//2, dtype=tf.int32)
 xsize = tf.random.uniform([2], minval=ERASE_TIME//2, maxval=ERASE_TIME, dtype=tf.int32)
 yoff = tf.random.uniform([2], minval=ERASE_MEL//2, maxval=HEIGHT-ERASE_MEL//2, dtype=tf.int32)
 ysize = tf.random.uniform([2], minval=ERASE_MEL//2, maxval=ERASE_MEL, dtype=tf.int32)
 image = tf.squeeze(image, axis=0)
 return image
if training:
gau = tf.keras.layers.GaussianNoise(0.3)
 image = tf.cond(tf.random.uniform([])< 0.5, lambda: gau(image, training=True), lambda: image)
 image = tf.image.random_brightness(image, 0.2)
 image = tf.image.random_flip_left_right(image)
 image = tf.cond(tf.random.uniform([])< 0.5, lambda: _specaugment(image), lambda: image)
 if test:
image = tf.cond(tf.random.uniform([])< 0.5, lambda: _specaugment(image), lambda: image)
 image =(image - tf.reduce_min(image)) /(tf.reduce_max(image)- tf.reduce_min(image)) * 255.0
image = tf.image.grayscale_to_rgb(image)
 image = cfg['model_params']['arch_preprocess'](image)
 return image
@tf.function
def _preprocess(x):
image = _preprocess_img(x['input'], training=True, test=False)
 return(image, x["target"])
 @tf.function
def _preprocess_val(x):
image = _preprocess_img(x['input'], training=False, test=False)
 return(image, x["target"])
 @tf.function
def _preprocess_test(x):
image = _preprocess_img(x['audio_spec'], training=False, test=True)
 return(image, x["recording_id"] )
%%time
if useXGB :
NFOLDS = 5
kf = StratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=42)
 y_preds_xgb2 = np.zeros(X_test.shape[0])
 y_oof_xgb2 = np.zeros(X_train.shape[0])
 score = 0
for fold,(tr_idx, val_idx)in enumerate(kf.split(X_train, y_train)) :
clf = xgb.XGBClassifier(
 n_estimators=1500,
max_depth=10,
learning_rate=0.05,
subsample=0.9,
colsample_bytree=0.9,
gamma = 0.2,
alpha = 4,
missing = -1,
tree_method='gpu_hist' 
)
 X_tr, X_vl = X_train.iloc[tr_idx, :], X_train.iloc[val_idx, :]
y_tr, y_vl = y_train.iloc[tr_idx], y_train.iloc[val_idx]
clf.fit(X_tr, y_tr)
 y_pred_train = clf.predict_proba(X_vl)[:,1]
y_oof_xgb2[val_idx] = y_pred_train
print("FOLD: ",fold,' AUC {}'.format(roc_auc_score(y_vl, y_pred_train)))
 score += roc_auc_score(y_vl, y_pred_train)/ NFOLDS
y_preds_xgb2+= clf.predict_proba(X_test)[:,1] / NFOLDS
del X_tr, X_vl, y_tr, y_vl
gc.collect()
y_preds_xgb2.reshape(-1,1)
 y_oof_xgb2.reshape(-1,1)
 print("
MEAN AUC = {}".format(score))
print("OOF AUC = {}".format(roc_auc_score(y_train, y_oof_xgb2)) )<init_hyperparams>
def create_model() :
def Classifier(shape_):
backbone = cfg['model_params']['arch'](( shape_), include_top=False, weights='imagenet')
 def cbr(x, out_layer, kernel, stride, dilation):
x = Conv2D(out_layer, kernel_size=kernel, dilation_rate=dilation, strides=stride, padding="same" )(x)
 x = BatchNormalization()(x)
 x = Activation("relu" )(x)
 return x
def wave_block(x, filters, kernel_size, n):
dilation_rates = [2**i for i in range(n)]
x = Conv2D(filters = filters,
kernel_size = 1,
padding = 'same' )(x)
 res_x = x
for dilation_rate in dilation_rates:
tanh_out = Conv2D(filters = filters,
kernel_size = kernel_size,
padding = 'same',
activation = 'tanh',
dilation_rate = dilation_rate )(x)
 sigm_out = Conv2D(filters = filters,
kernel_size = kernel_size,
padding = 'same',
activation = 'sigmoid',
dilation_rate = dilation_rate )(x)
 x = Multiply()([tanh_out, sigm_out])
 x = Conv2D(filters = filters,
kernel_size = 1,
padding = 'same' )(x)
 res_x = Add()([res_x, x])
 return res_x
def wavenet(layer):
x = cbr(layer, 192, 7, 1, 1)
 x = BatchNormalization()(x)
 x = wave_block(x, 192, 3, 1)
 x = cbr(x, 96, 7, 1, 1)
 x = BatchNormalization()(x)
 x = wave_block(x, 96, 3, 1)
 x = cbr(x, 48, 5, 1, 1)
 x = BatchNormalization()(x)
 x = wave_block(x, 48, 3, 1)
 return x
def wavenet1(layer):
x = cbr(layer, 4, 7, 1, 1)
 x = BatchNormalization()(x)
 x = wave_block(x, 3, 3, 1)
 x = cbr(x, 3, 7, 1, 1)
 x = BatchNormalization()(x)
 x = wave_block(x, 16, 3, 1)
 x = cbr(x, 3, 5, 1, 1)
 return x
x0 = backbone
print('1')
 x1 = tf.keras.layers.GlobalAveragePooling2D()(x0.layers[-1].output)
 x3 = tf.keras.layers.GlobalAveragePooling2D()(x0.layers[-7].output)
 x5 = tf.keras.layers.GlobalAveragePooling2D()(x0.layers[-18].output)
 print('2')
 x1=wavenet(x0.layers[-1].output)
 x3=wavenet(x0.layers[-7].output)
 x5=wavenet(x0.layers[-18].output)
 x1 = tf.keras.layers.GlobalAveragePooling2D()(x1)
 x3 = tf.keras.layers.GlobalAveragePooling2D()(x3)
 x5 = tf.keras.layers.GlobalAveragePooling2D()(x5)
 print('4')
 x = tf.concat([x1,x3,x5],axis = 1)
 x = tf.keras.layers.Dropout(0.7 )(x)
 x = tf.keras.layers.Dense(192 )(x)
 x = tf.keras.layers.Dropout(0.4 )(x)
 output = tf.keras.layers.Softmax(dtype='float32' )(x)
 output =tf.keras.layers.Dense(CLASS_N )(x)
 print('5')
 model = tf.keras.models.Model(inputs = x0.input, outputs = output)
 return model
return Classifier([HEIGHT,WIDTH,3])
 model = create_model()
model.summary()
param = {'num_leaves': 30,
'min_data_in_leaf': 30,
'objective':'binary',
'max_depth': -1,
'learning_rate': 0.01,
"boosting": "gbdt",
"feature_fraction": 0.8,
"bagging_freq": 1,
"bagging_fraction": 0.8 ,
"bagging_seed": 11,
"metric": 'auc',
"lambda_l1": 0.1,
"random_state": 133,
"verbosity": -1}<set_options>
@tf.function
def _mixup(inp, targ):
indice = tf.range(len(inp))
indice = tf.random.shuffle(indice)
 sinp = tf.gather(inp, indice, axis=0)
 starg = tf.gather(targ, indice, axis=0)
 alpha = 0.2
t = tf.compat.v1.distributions.Beta(alpha, alpha ).sample([len(inp)])
 tx = tf.reshape(t, [-1, 1, 1, 1])
 ty = tf.reshape(t, [-1, 1])
 x = inp * tx + sinp *(1-tx)
 y = targ * ty + starg *(1-ty)
 return x, y
max_iter = 5
gc.collect()<split>
tfrecs = sorted(tf.io.gfile.glob(TRAIN_TFREC + '/*.tfrec'))
parsed_trainval =(tf.data.TFRecordDataset(tfrecs, num_parallel_reads=AUTOTUNE)
.map(_parse_function, num_parallel_calls=AUTOTUNE ).unbatch() 
.filter(_filtTP ).enumerate() )
%%time
if useLGBM :
folds = TimeSeriesSplit(n_splits =5)
 oof = np.zeros(len(X_train))
features = [c for c in X_train.columns]
predictions = np.zeros(len(X_test))
start = time.time()
feature_importance_df = pd.DataFrame()
start_time= time.time()
score = [0 for _ in range(folds.n_splits)]
for fold_,(trn_idx, val_idx)in enumerate(folds.split(X_train, y_train)) :
print("fold n°{}".format(fold_))
trn_data = lgb.Dataset(X_train.iloc[trn_idx][features],
label=y_train.iloc[trn_idx] 
)
 val_data = lgb.Dataset(X_train.iloc[val_idx][features],
label=y_train.iloc[val_idx] 
)
 num_round = 500
clf = lgb.train(param,
trn_data,
num_round,
valid_sets = [trn_data, val_data],
verbose_eval=10,
early_stopping_rounds = 100)
 oof[val_idx] = clf.predict(X_train.iloc[val_idx][features], num_iteration=clf.best_iteration)
 fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = features
fold_importance_df["importance"] = clf.feature_importance(importance_type='gain')
 fold_importance_df["fold"] = fold_ + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 initial_idx = 0
chunk_size = 100000
current_pred = np.zeros(len(X_test))
while initial_idx < X_test.shape[0]:
final_idx = min(initial_idx + chunk_size, X_test.shape[0])
 idx = range(initial_idx, final_idx)
 current_pred[idx] = clf.predict(X_test.iloc[idx][features], num_iteration=clf.best_iteration)
 initial_idx = final_idx
predictions += current_pred / min(folds.n_splits, max_iter)
 print("time elapsed: {:<5.2}s".format(( time.time() - start_time)/ 3600))
score[fold_] = metrics.roc_auc_score(y_train.iloc[val_idx], oof[val_idx])
 if fold_ == max_iter - 1: break
if(folds.n_splits == max_iter):
print("CV score: {:<8.5f}".format(metrics.roc_auc_score(y_train, oof)))
 else:
print("CV score: {:<8.5f}".format(sum(score)/ max_iter)) 
<count_values>
indices = []
spid = []
recid = []
for i, sample in tqdm(parsed_trainval.prefetch(AUTOTUNE)) :
indices.append(i.numpy())
 spid.append(sample['species_id'].numpy())
 recid.append(sample['recording_id'].numpy().decode() )
feature_importance_df['fold'].value_counts()<sort_values>
table = pd.DataFrame({'indices': indices, 'species_id': spid, 'recording_id': recid})
 table
df_lose_it = feature_importance_df.loc[feature_importance_df['fold'] == 5].sort_values(by="importance", ascending=True ).head(100 )<drop_column>
def create_idx_filter(indice):
@tf.function
def _filt(i, x):
return tf.reduce_any(indice == i)
 return _filt
@tf.function
def _remove_idx(i, x):
return x
df_lose_it=df_lose_it.loc[df_lose_it['importance'] == 0.0000]
drop_add = []
for vals in df_lose_it['feature'] :
drop_add.append(vals)
<drop_column>
def create_train_dataset(batchsize, train_idx):
global parsed_trainval
parsed_train =(parsed_trainval 
.filter(create_idx_filter(train_idx)) 
.map(_remove_idx))
dataset =(parsed_train.cache() 
.shuffle(len(train_idx)) 
.repeat() 
.map(_cut_wav, num_parallel_calls=AUTOTUNE)
.map(_wav_to_spec, num_parallel_calls=AUTOTUNE)
.map(_create_annot, num_parallel_calls=AUTOTUNE)
.map(_preprocess, num_parallel_calls=AUTOTUNE)
.batch(batchsize))
if cfg['model_params']['mixup']:
dataset =(dataset.map(_mixup, num_parallel_calls=AUTOTUNE)
.prefetch(AUTOTUNE))
else:
dataset = dataset.prefetch(AUTOTUNE)
 return dataset
def create_val_dataset(batchsize, val_idx):
global parsed_trainval
parsed_val =(parsed_trainval 
.filter(create_idx_filter(val_idx)) 
.map(_remove_idx))
vdataset =(parsed_val 
.map(_cut_wav_val, num_parallel_calls=AUTOTUNE)
.map(_wav_to_spec, num_parallel_calls=AUTOTUNE)
.map(_create_annot, num_parallel_calls=AUTOTUNE)
.map(_preprocess_val, num_parallel_calls=AUTOTUNE)
.batch(8*tpu_strategy.num_replicas_in_sync)
.cache())
 return vdataset
X_train.drop(drop_add,axis=1, inplace=True)
 X_test.drop(drop_add, axis=1, inplace=True)
 X_train.head()<split>
@tf.function
def _one_sample_positive_class_precisions(example):
y_true, y_pred = example
retrieved_classes = tf.argsort(y_pred, direction='DESCENDING')
 class_rankings = tf.argsort(retrieved_classes)
 retrieved_class_true = tf.gather(y_true, retrieved_classes)
 retrieved_cumulative_hits = tf.math.cumsum(tf.cast(retrieved_class_true, tf.float32))
idx = tf.where(y_true)[:, 0]
i = tf.boolean_mask(class_rankings, y_true)
 r = tf.gather(retrieved_cumulative_hits, i)
 c = 1 + tf.cast(i, tf.float32)
 precisions = r / c
dense = tf.scatter_nd(idx[:, None], precisions, [y_pred.shape[0]])
 return dense
class LWLRAP(tf.keras.metrics.Metric):
def __init__(self, num_classes, name='lwlrap'):
super().__init__(name=name)
 self._precisions = self.add_weight(
 name='per_class_cumulative_precision',
shape=[num_classes],
initializer='zeros', 
)
 self._counts = self.add_weight(
 name='per_class_cumulative_count',
shape=[num_classes],
initializer='zeros', 
)
 def update_state(self, y_true, y_pred, sample_weight=None):
precisions = tf.map_fn(
 fn=_one_sample_positive_class_precisions,
elems=(y_true, y_pred),
dtype=(tf.float32), 
)
 increments = tf.cast(precisions > 0, tf.float32)
 total_increments = tf.reduce_sum(increments, axis=0)
 total_precisions = tf.reduce_sum(precisions, axis=0)
 self._precisions.assign_add(total_precisions)
 self._counts.assign_add(total_increments)
 def result(self):
per_class_lwlrap = self._precisions / tf.maximum(self._counts, 1.0)
 per_class_weight = self._counts / tf.reduce_sum(self._counts)
 overall_lwlrap = tf.reduce_sum(per_class_lwlrap * per_class_weight)
 return overall_lwlrap
def reset_states(self):
self._precisions.assign(self._precisions * 0)
 self._counts.assign(self._counts * 0 )
%%time
if useLGBM :
folds = TimeSeriesSplit(n_splits =5)
 oof = np.zeros(len(X_train))
features = [c for c in X_train.columns]
predictions = np.zeros(len(X_test))
start = time.time()
feature_importance_df = pd.DataFrame()
start_time= time.time()
score = [0 for _ in range(folds.n_splits)]
for fold_,(trn_idx, val_idx)in enumerate(folds.split(X_train, y_train)) :
print("fold n°{}".format(fold_))
trn_data = lgb.Dataset(X_train.iloc[trn_idx][features],
label=y_train.iloc[trn_idx] 
)
 val_data = lgb.Dataset(X_train.iloc[val_idx][features],
label=y_train.iloc[val_idx] 
)
 num_round = 10000
clf = lgb.train(param,
trn_data,
num_round,
valid_sets = [trn_data, val_data],
verbose_eval=200,
early_stopping_rounds = 200)
 oof[val_idx] = clf.predict(X_train.iloc[val_idx][features], num_iteration=clf.best_iteration)
 fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = features
fold_importance_df["importance"] = clf.feature_importance(importance_type='gain')
 fold_importance_df["fold"] = fold_ + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 initial_idx = 0
chunk_size = 100000
current_pred = np.zeros(len(X_test))
while initial_idx < X_test.shape[0]:
final_idx = min(initial_idx + chunk_size, X_test.shape[0])
 idx = range(initial_idx, final_idx)
 current_pred[idx] = clf.predict(X_test.iloc[idx][features], num_iteration=clf.best_iteration)
 initial_idx = final_idx
predictions += current_pred / min(folds.n_splits, max_iter)
 print("time elapsed: {:<5.2}s".format(( time.time() - start_time)/ 3600))
score[fold_] = metrics.roc_auc_score(y_train.iloc[val_idx], oof[val_idx])
 if fold_ == max_iter - 1: break
if(folds.n_splits == max_iter):
print("CV score: {:<8.5f}".format(metrics.roc_auc_score(y_train, oof)))
 else:
print("CV score: {:<8.5f}".format(sum(score)/ max_iter)) 
<save_to_csv>
def _parse_function_test(example_proto):
sample = tf.io.parse_single_example(example_proto, feature_description)
 wav, _ = tf.audio.decode_wav(sample['audio_wav'], desired_channels=1)
 @tf.function
def _cut_audio(i):
_sample = {
'audio_wav': tf.reshape(wav[i*SR*TIME:(i+1)*SR*TIME], [SR*TIME]),
'recording_id': sample['recording_id']
}
return _sample
return tf.map_fn(_cut_audio, tf.range(60//TIME), dtype={
'audio_wav': tf.float32,
'recording_id': tf.string
})
 def inference(model):
tdataset =(tf.data.TFRecordDataset(tf.io.gfile.glob(TEST_TFREC + '/*.tfrec'), num_parallel_reads=AUTOTUNE)
.map(_parse_function_test, num_parallel_calls=AUTOTUNE ).unbatch() 
.map(_wav_to_spec, num_parallel_calls=AUTOTUNE)
.map(_preprocess_test, num_parallel_calls=AUTOTUNE)
.batch(128*(60//TIME)).prefetch(AUTOTUNE))
rec_ids = []
probs = []
for inp, rec_id in tqdm(tdataset):
with tpu_strategy.scope() :
pred = model.predict_on_batch(tf.reshape(inp, [-1, HEIGHT, WIDTH, 3]))
prob = tf.sigmoid(pred)
 prob = tf.reduce_max(tf.reshape(prob, [-1, 60//TIME, CLASS_N]), axis=1)
 rec_id_stack = tf.reshape(rec_id, [-1, 60//TIME])
 for rec in rec_id.numpy() :
assert len(np.unique(rec)) == 1
rec_ids.append(rec_id_stack.numpy() [:,0])
 probs.append(prob.numpy())
 crec_ids = np.concatenate(rec_ids)
 cprobs = np.concatenate(probs)
 sub = pd.DataFrame({
'recording_id': list(map(lambda x: x.decode() , crec_ids.tolist())) ,
**{f's{i}': cprobs[:,i] for i in range(CLASS_N)}
})
 sub = sub.sort_values('recording_id')
 return sub
sample_submission['isFraud'] = predictions
sample_submission.to_csv('simple_ensemble.csv' )<load_from_csv>
def train_and_inference(splits, split_id):
print(split_id)
 batchsize = cfg['model_params']['batchsize_per_tpu'] * tpu_strategy.num_replicas_in_sync
print("batchsize", batchsize)
 loss_fn = cfg['model_params']['loss']['fn'](from_logits=True, **cfg['model_params']['loss']['params'])
 idx_train_tf = tf.constant(splits[split_id][0])
 idx_val_tf = tf.constant(splits[split_id][1])
 dataset = create_train_dataset(batchsize, idx_train_tf)
 vdataset = create_val_dataset(batchsize, idx_val_tf)
 optimizer = cfg['model_params']['optim']['fn'](**cfg['model_params']['optim']['params'])
 with tpu_strategy.scope() :
model = create_model()
model.compile(optimizer=optimizer, loss=loss_fn, metrics=[LWLRAP(CLASS_N)])
 if split_id not in(10,10):
history = model.fit(dataset,
steps_per_epoch=cfg['model_params']['iteration_per_epoch'],
epochs=cfg['model_params']['epoch'],
validation_data=vdataset,
callbacks=[
tf.keras.callbacks.ReduceLROnPlateau(
 'val_lwlrap', patience=10 
),
tf.keras.callbacks.ModelCheckpoint(
 filepath=models_path+'model_best_%d.h5' % split_id,
save_weights_only=True,
monitor='val_lwlrap',
mode='max',
save_best_only=True),
] 
)
 plot_history(history, 'history_%d.png' % split_id)
 best_score = max(history.history['val_lwlrap'])
 print(best_score)
 model.load_weights(models_path+'model_best_%d.h5' % split_id)
 sub=inference(model)
 del model
gc.collect()
return sub,best_score
<drop_column><EOS>