Kaggle competition: Rainforest Connection Species Audio Detection
<SOS> metric: WeightedLabelRankingAveragePrecision Kaggle data source: rainforest-connection-species-audio-detection<define_variables>
! pip install -q efficientnet
PATH = './'
TRAIN = '.. /input/airbus-ship-detection/train_v2/'
TEST = '.. /input/airbus-ship-detection/test_v2/'
SEGMENTATION = '.. /input/airbus-ship-detection/train_ship_segmentations_v2.csv'
PRETRAINED_DETECTION_PATH = '.. /input/fine-tuning-resnet34-on-ship-detection/models/'
PRETRAINED_SEGMENTATION_PATH = '.. /input/unet34-dice-0-87/models/'
DETECTION_TEST_PRED = '.. /input/fine-tuning-resnet34-ship-redo/Have_ship_or_not.csv'<choose_model_class>
TRAIN = True
if TRAIN:
MODEL_DIR = "."
else:
MODEL_DIR = "/kaggle/input/rfcx-train-fp-weights"
nw = 2
arch = resnet34<split>
import math, os, re, warnings, random
import tensorflow as tf
import numpy as np
import pandas as pd
import librosa
from kaggle_datasets import KaggleDatasets
import matplotlib.pyplot as plt
from IPython.display import Audio
from tensorflow.keras import Model, layers
from sklearn.model_selection import KFold
import tensorflow.keras.backend as K
from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, LearningRateScheduler
from tensorflow.keras.layers import GlobalAveragePooling2D, GlobalMaxPooling2D, Input, Dense, Dropout, GaussianNoise, Conv2D, SpatialDropout2D, Lambda
from tensorflow.keras.applications import ResNet50
import tensorflow_addons as tfa
import efficientnet.keras as efn
import seaborn as sns
from functools import partial
train_names = [f for f in os.listdir(TRAIN)][:1000]
test_names = [f for f in os.listdir(TEST)][:1000]
tr_n, val_n = train_test_split(train_names, test_size=0.05, random_state=42)
 segmentation_df = pd.read_csv(os.path.join(PATH, SEGMENTATION)).set_index('ImageId')
 def cut_empty(names):
return [name for name in names
if(type(segmentation_df.loc[name]['EncodedPixels'])!= float)]
tr_n_cut = cut_empty(tr_n)
 val_n_cut = cut_empty(val_n )<categorify>
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print(f'Running on TPU {tpu.master() }')
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
AUTO = tf.data.experimental.AUTOTUNE
REPLICAS = strategy.num_replicas_in_sync
print(f'REPLICAS: {REPLICAS}' )
def get_mask(img_id, df):
shape =(768,768)
 img = np.zeros(shape[0]*shape[1], dtype=np.uint8)
 masks = df.loc[img_id]['EncodedPixels']
if(type(masks)== float): return img.reshape(shape)
 if(type(masks)== str): masks = [masks]
for mask in masks:
s = mask.split()
for i in range(len(s)//2):
start = int(s[2*i])- 1
length = int(s[2*i+1])
 img[start:start+length] = 1
return img.reshape(shape ).T<prepare_x_and_y>
def seed_everything(seed=0):
random.seed(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 os.environ['TF_DETERMINISTIC_OPS'] = '1'
seed = 42
seed_everything(seed)
 warnings.filterwarnings('ignore' )
class pdFilesDataset(FilesDataset):
def __init__(self, fnames, path, transform):
self.segmentation_df = pd.read_csv(SEGMENTATION ).set_index('ImageId')
 super().__init__(fnames, transform, path)
 def get_x(self, i):
img = open_image(os.path.join(self.path, self.fnames[i]))
if self.sz == 768: return img
else: return cv2.resize(img,(self.sz, self.sz))
def get_y(self, i):
mask = np.zeros(( 768,768), dtype=np.uint8)if(self.path == TEST)\
else get_mask(self.fnames[i], self.segmentation_df)
 img = Image.fromarray(mask ).resize(( self.sz, self.sz)).convert('RGB')
 return np.array(img ).astype(np.float32)
 def get_c(self): return 0<create_dataframe>
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1)) for filename in filenames]
return np.sum(n)
 TRAIN_DATA_DIR = 'rfcx-audio-detection'
TRAIN_GCS_PATH = KaggleDatasets().get_gcs_path(TRAIN_DATA_DIR)
 TEST_DATA_DIR = 'rfcx-species-audio-detection'
TEST_GCS_PATH = KaggleDatasets().get_gcs_path(TEST_DATA_DIR)
 TEST_FILES = tf.io.gfile.glob(TEST_GCS_PATH + '/tfrecords/test/*.tfrec')
 FILENAMES = tf.io.gfile.glob(TEST_GCS_PATH + '/tfrecords/train/*.tfrec')
 no_of_training_samples = count_data_items(FILENAMES)
 print('num_training_samples are', no_of_training_samples)
def get_data(sz,bs):
tfms = tfms_from_model(arch, sz, crop_type=CropType.NO, tfm_y=TfmType.CLASS)
 tr_names = tr_n if(len(tr_n_cut)%bs == 0)else tr_n[:-(len(tr_n_cut)%bs)]
ds = ImageData.get_ds(pdFilesDataset,(tr_names,TRAIN), 
(val_n_cut,TRAIN), tfms, test=(test_names,TEST))
md = ImageData(PATH, ds, bs, num_workers=nw, classes=None)
 return md<find_best_params>
cut,lr_cut = model_meta[arch]<choose_model_class>
dd = defaultdict(list )
def get_base(pre=True):
layers = cut_model(arch(pre), cut)
 return nn.Sequential(*layers )<concatenate>
GLOBAL_BATCH_SIZE = 4 * REPLICAS
STEPS_PER_EPOCH = no_of_training_samples // GLOBAL_BATCH_SIZE
print(STEPS_PER_EPOCH )
class UnetBlock(nn.Module):
def __init__(self, up_in, x_in, n_out):
super().__init__()
up_out = x_out = n_out//2
self.x_conv = nn.Conv2d(x_in, x_out, 1)
 self.tr_conv = nn.ConvTranspose2d(up_in, up_out, 2, stride=2)
 self.bn = nn.BatchNorm2d(n_out)
 def forward(self, up_p, x_p):
up_p = self.tr_conv(up_p)
 x_p = self.x_conv(x_p)
 cat_p = torch.cat([up_p,x_p], dim=1)
 return self.bn(F.relu(cat_p))
class SaveFeatures() :
features=None
def __init__(self, m): self.hook = m.register_forward_hook(self.hook_fn)
 def hook_fn(self, module, input, output): self.features = output
def remove(self): self.hook.remove()
class Unet34(nn.Module):
def __init__(self, rn):
super().__init__()
self.rn = rn
self.sfs = [SaveFeatures(rn[i])for i in [2,4,5,6]]
self.up1 = UnetBlock(512,256,256)
 self.up2 = UnetBlock(256,128,256)
 self.up3 = UnetBlock(256,64,256)
 self.up4 = UnetBlock(256,64,256)
 self.up5 = nn.ConvTranspose2d(256, 1, 2, stride=2)
 def forward(self,x):
x = F.relu(self.rn(x))
x = self.up1(x, self.sfs[3].features)
 x = self.up2(x, self.sfs[2].features)
 x = self.up3(x, self.sfs[1].features)
 x = self.up4(x, self.sfs[0].features)
 x = self.up5(x)
 return x[:,0]
def close(self):
for sf in self.sfs: sf.remove()
class UnetModel() :
def __init__(self,model,name='Unet'):
self.model,self.name = model,name
def get_layer_groups(self, precompute):
lgs = list(split_by_idxs(children(self.model.rn), [lr_cut]))
return lgs + [children(self.model)[1:]]<categorify>
CUT = 5
STRIDE = 3
EPOCHS = 25
AUG_BATCH = GLOBAL_BATCH_SIZE
VAL_BATCH_SIZE = GLOBAL_BATCH_SIZE
LEARNING_RATE = 0.0015
WARMUP_LEARNING_RATE = 1e-5
WARMUP_EPOCHS = int(EPOCHS*0.1)
 PATIENCE = EPOCHS
N_FOLDS = 4
NUM_TRAINING_SAMPLES = no_of_training_samples
class params:
sample_rate = 48000
stft_window_seconds: float = 0.025
stft_hop_seconds: float = 0.005
frame_length: int = 1200
mel_bands: int = 512
mel_min_hz: float = 50.0
mel_max_hz: float = 24000.0
log_offset: float = 0.001
patch_window_seconds: float = 0.96
patch_hop_seconds: float = 0.48
patch_frames = int(round(patch_window_seconds / stft_hop_seconds))
patch_bands = mel_bands
height = mel_bands
width = CUT * 200
num_classes: int = 24
dropout = 0.35
classifier_activation: str = 'sigmoid'
IMAGE_SIZE = [params.height, params.width]
CLASSES = params.num_classes
def split_mask(mask):
threshold = 0.57777778
threshold_obj = 30
labled,n_objs = ndimage.label(mask > threshold)
 result = []
for i in range(n_objs):
obj =(labled == i + 1 ).astype(int)
 if(obj.sum() > threshold_obj): result.append(obj)
 return result
def get_mask_ind(img_id, df, shape =(768,768)) :
masks = df.loc[img_id]['EncodedPixels']
if(type(masks)== float): return []
if(type(masks)== str): masks = [masks]
result = []
for mask in masks:
img = np.zeros(shape[0]*shape[1], dtype=np.uint8)
 s = mask.split()
for i in range(len(s)//2):
start = int(s[2*i])- 1
length = int(s[2*i+1])
 img[start:start+length] = 1
result.append(img.reshape(shape ).T)
 return result<concatenate>
feature_description = {
'wav': tf.io.FixedLenFeature([], tf.string),
'recording_id': tf.io.FixedLenFeature([], tf.string),
'target' : tf.io.FixedLenFeature([], tf.float32),
'song_id': tf.io.FixedLenFeature([], tf.float32),
'tmin' : tf.io.FixedLenFeature([], tf.float32),
'fmin' : tf.io.FixedLenFeature([], tf.float32),
'tmax' : tf.io.FixedLenFeature([], tf.float32),
'fmax' : tf.io.FixedLenFeature([], tf.float32),
'is_tp' : tf.io.FixedLenFeature([], tf.int32)
 }
feature_dtype = {
'wav': tf.float32,
'recording_id': tf.string,
'target': tf.float32,
'song_id': tf.float32,
't_min': tf.float32,
'f_min': tf.float32,
't_max': tf.float32,
'f_max':tf.float32,
}
def aug_unit(x,fwd=True,mask=False):
return x
def aug_flipV(x,fwd=True,mask=False):
return x.flip(2)if mask else x.flip(3)
 def aug_flipH(x,fwd=True,mask=False):
return x.flip(1)if mask else x.flip(2)
 def aug_T(x,fwd=True,mask=False):
return torch.transpose(x,1,2)if mask else torch.transpose(x,2,3)
 def aug_rot_2(x,fwd=True,mask=False):
return aug_flipV(aug_flipH(x,fwd,mask),fwd,mask)
 def aug_rot_4cr(x,fwd=True,mask=False):
return aug_flipV(aug_T(x,fwd,mask),fwd,mask)if fwd else \
aug_T(aug_flipV(x,fwd,mask),fwd,mask)
 def aug_rot_4cw(x,fwd=True,mask=False):
return aug_flipH(aug_T(x,fwd,mask),fwd,mask)if fwd else \
aug_T(aug_flipH(x,fwd,mask),fwd,mask)
 def aug_rot_2T(x,fwd=True,mask=False):
return aug_rot_2(aug_T(x,fwd,mask),fwd,mask)
 trms_side_on = [aug_unit,aug_flipH]
trms_top_down = [aug_unit,aug_flipV]
trms_dihedral = [aug_unit,aug_flipH,aug_flipV,aug_T,aug_rot_2,aug_rot_2T,
aug_rot_4cw,aug_rot_4cr]
def enc_img(img):
return torch.transpose(torch.tensor(img),0,2 ).unsqueeze(0)
 def dec_img(img):
return to_np(torch.transpose(img.squeeze(0),0,2))
def display_augs(x,augs=aug_unit):
columns = 4
n = len(augs)
 rows = n//4 + 1
fig=plt.figure(figsize=(columns*4, rows*4))
img = enc_img(x)
 for i in range(rows):
for j in range(columns):
idx = j+i*columns
if idx >= n: break
fig.add_subplot(rows, columns, idx+1)
 plt.axis('off')
 plt.imshow(dec_img(augs[idx](img)))
 plt.show()
img = np.array(Image.open(os.path.join(TRAIN,'ce69faa4b.jpg')))
 display_augs(img,trms_dihedral )<choose_model_class>
def waveform_to_log_mel_spectrogram(waveform,target_or_rec_id):

 window_length_samples = int(
 round(params.sample_rate * params.stft_window_seconds))
hop_length_samples = int(
 round(params.sample_rate * params.stft_hop_seconds))
fft_length = 2 ** int(np.ceil(np.log(window_length_samples)/ np.log(2.0)))
 num_spectrogram_bins = fft_length // 2 + 1
magnitude_spectrogram = tf.abs(tf.signal.stft(
 signals=waveform,
frame_length=params.frame_length,
frame_step=hop_length_samples,
fft_length= fft_length))
linear_to_mel_weight_matrix = tf.signal.linear_to_mel_weight_matrix(
 num_mel_bins=params.mel_bands,
num_spectrogram_bins=num_spectrogram_bins,
sample_rate=params.sample_rate,
lower_edge_hertz=params.mel_min_hz,
upper_edge_hertz=params.mel_max_hz)
 mel_spectrogram = tf.matmul(
 magnitude_spectrogram, linear_to_mel_weight_matrix)
 log_mel = tf.math.log(mel_spectrogram + params.log_offset)
 log_mel = tf.transpose(log_mel)
 log_mel_spectrogram = tf.reshape(log_mel , [tf.shape(log_mel)[0] ,tf.shape(log_mel)[1],1])
 spectrogram_hop_length_samples = int(
 round(params.sample_rate * params.stft_hop_seconds))
spectrogram_sample_rate = params.sample_rate / spectrogram_hop_length_samples
patch_window_length_samples = int(
 round(spectrogram_sample_rate * params.patch_window_seconds))
patch_hop_length_samples = int(
 round(spectrogram_sample_rate * params.patch_hop_seconds))
features = tf.signal.frame(
 signal=log_mel_spectrogram,
frame_length=patch_window_length_samples,
frame_step=patch_hop_length_samples,
axis=0)
 return log_mel_spectrogram, target_or_rec_id
m = to_gpu(Unet34(get_base(False)))
 models = UnetModel(m)
 sz = 768
bs = 1
md = get_data(sz,bs)
 learn = ConvLearner(md, models)
 learn.models_path = PRETRAINED_SEGMENTATION_PATH
learn.load('Unet34_768_1')
 learn.models_path = PATH<set_options>
def frequency_masking(mel_spectrogram):
frequency_masking_para = 80,
frequency_mask_num = 2
fbank_size = tf.shape(mel_spectrogram)
 n, v = fbank_size[0], fbank_size[1]
for i in range(frequency_mask_num):
f = tf.random.uniform([], minval=0, maxval= tf.squeeze(frequency_masking_para), dtype=tf.int32)
 v = tf.cast(v, dtype=tf.int32)
 f0 = tf.random.uniform([], minval=0, maxval= tf.squeeze(v-f), dtype=tf.int32)
 mask = tf.concat(( tf.ones(shape=(n, v - f0 - f,1)) ,
tf.zeros(shape=(n, f,1)) ,
tf.ones(shape=(n, f0,1)) , 
),1)
 mel_spectrogram = mel_spectrogram * mask
return tf.cast(mel_spectrogram, dtype=tf.float32)
 def time_masking(mel_spectrogram):
time_masking_para = 40,
time_mask_num = 1
fbank_size = tf.shape(mel_spectrogram)
 n, v = fbank_size[0], fbank_size[1]
for i in range(time_mask_num):
t = tf.random.uniform([], minval=0, maxval=tf.squeeze(time_masking_para), dtype=tf.int32)
 t0 = tf.random.uniform([], minval=0, maxval= n-t, dtype=tf.int32)
 mask = tf.concat(( tf.ones(shape=(n-t0-t, v,1)) ,
tf.zeros(shape=(t, v,1)) ,
tf.ones(shape=(t0, v,1)) , 
), 0)
 mel_spectrogram = mel_spectrogram * mask
return tf.cast(mel_spectrogram, dtype=tf.float32)
 def random_brightness(image):
return tf.image.random_brightness(image, 0.2)
 def random_gamma(image):
return tf.image.random_contrast(image, lower = 0.1, upper = 0.3)
 def random_flip_right(image):
return tf.image.random_flip_left_right(image)
 def random_flip_up_down(image):
return tf.image.random_flip_left_right(image)
 def shift_scale_shear(image):
a0 = tf.random.uniform([], minval=0.9, maxval=1.1)
 a1 = tf.random.uniform([], minval=-0.05, maxval=0.05)
 a2 = tf.random.uniform([], minval=-params.width * 0.1, maxval = 0.1 * params.width)
 b0 = tf.random.uniform([], minval=-0.05, maxval=0.05)
 b1 = tf.random.uniform([], minval=0.9, maxval=1.1)
 b2 = tf.random.uniform([], minval=-params.height * 0.1, maxval = 0.1 * params.height)
 return tfa.image.transform(image, [a0, a1, a2, b0, b1, b2, 0, 0], interpolation="bilinear", fill_mode="reflect")
 def random_rotate(image):
angle = tf.random.uniform([], minval=-10, maxval=10)
 return tfa.image.rotate(image, angle, interpolation="nearest", fill_mode="reflect")
 available_ops = [
frequency_masking ,
time_masking,
random_brightness,
random_flip_up_down,
random_flip_right,
shift_scale_shear,
random_rotate
]
def apply_augmentation(image, target):
num_layers = int(np.random.uniform(low = 0, high = 3))
for layer_num in range(num_layers):
op_to_select = tf.random.uniform([], maxval=len(available_ops), dtype=tf.int32, seed = seed)
 for(i, op_name)in enumerate(available_ops):
image = tf.cond(
 tf.equal(i, op_to_select),
lambda selected_func=op_name,: selected_func(
 image),
lambda: image)
 return image, target
config = ktf.ConfigProto()
config.gpu_options.per_process_gpu_memory_fraction = 0.4
config.gpu_options.visible_device_list = "0"
set_session(ktf.Session(config=config))
print('ktf limited' )<define_variables>
def string_split_semicolon(column):
split_labels_sc = tf.strings.split(column, sep=';')
 return split_labels_sc
def string_split_comma(column):
split_labels_c = tf.strings.split(column, sep=',')
 return split_labels_c
def get_label_info(label_info):
first_split = string_split_semicolon(label_info)
 remove_quotes = tf.strings.regex_replace(first_split, '"', "")
 label_info = string_split_comma(remove_quotes)
 return label_info
montage_rgb = lambda x: np.stack([montage(x[:, :, :, i])for i in range(x.shape[3])], -1)
 ship_dir = '.. /input/airbus-ship-detection/'
train_image_dir = os.path.join(ship_dir, 'train_v2')
 test_image_dir = os.path.join(ship_dir, 'test_v2')
 exclude_list = ['6384c3e78.jpg','13703f040.jpg', '14715c06d.jpg', '33e0ff2d5.jpg',
'4d4e09f2a.jpg', '877691df8.jpg', '8b909bb20.jpg', 'a8d99130e.jpg',
'ad55c3143.jpg', 'c8260c541.jpg', 'd6c7f17c7.jpg', 'dc3e7c901.jpg',
'e44dffe88.jpg', 'ef87bad36.jpg', 'f083256d8.jpg']
def multi_rle_encode(img, **kwargs):

 labels = label(img)
 if img.ndim > 2:
return [rle_encode(np.sum(labels==k, axis=2), **kwargs)for k in np.unique(labels[labels>0])]
else:
return [rle_encode(labels==k, **kwargs)for k in np.unique(labels[labels>0])]
def rle_encode(img):

 pixels = img.T.flatten()
pixels = np.concatenate([[0], pixels, [0]])
 runs = np.where(pixels[1:] != pixels[:-1])[0] + 1
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs)
 def rle_decode(mask_rle, shape=(768, 768)) :

 s = mask_rle.split()
starts, lengths = [np.asarray(x, dtype=int)for x in(s[0:][::2], s[1:][::2])]
starts -= 1
ends = starts + lengths
img = np.zeros(shape[0]*shape[1], dtype=np.uint8)
 for lo, hi in zip(starts, ends):
img[lo:hi] = 1
return img.reshape(shape ).T
def masks_as_image(in_mask_list):
all_masks = np.zeros(( 768, 768), dtype = np.int16)
 for mask in in_mask_list:
if isinstance(mask, str):
all_masks += rle_decode(mask)
 return np.expand_dims(all_masks, -1 )<feature_engineering>
def read_original_tfrecord(example):
TFREC_FORMAT = {
'audio_wav': tf.io.FixedLenFeature([], tf.string),
'recording_id': tf.io.FixedLenFeature([], tf.string),
'label_info': tf.io.FixedLenFeature([], tf.string, default_value='-1,-1,0,0,0,0,1'),
}
example = tf.io.parse_single_example(example, TFREC_FORMAT)
 audio = example['audio_wav']
label_info = get_label_info(example['label_info'])
 species_id = tf.strings.to_number(tf.gather_nd(label_info, [0, 0]), tf.int32)
 tmin = tf.strings.to_number(tf.gather_nd(label_info, [0, 2]))
fmin = tf.strings.to_number(tf.gather_nd(label_info, [0, 3]))
tmax = tf.strings.to_number(tf.gather_nd(label_info, [0, 4]))
fmax = tf.strings.to_number(tf.gather_nd(label_info, [0, 5]))
is_tp = tf.strings.to_number(tf.gather_nd(label_info, [0, 6]), tf.int32)
 features = {'wav': audio,
'recording_id': example['recording_id'],
'target': species_id,
'tmin': tmin,
'tmax': tmax,
'fmin': fmin,
'fmax': fmax,
'is_tp': is_tp
}
return features
masks = pd.read_csv(os.path.join('.. /input/airbus-ship-detection/',
'train_ship_segmentations_v2.csv')).head(5000)
 masks['corrupted'] = masks['ImageId'].map(lambda x: 1 if x in exclude_list else 0)
 print(masks['corrupted'].value_counts())
 masks = masks[masks['corrupted'] == 0]
masks = masks.drop(['corrupted'], axis=1)
 print(masks.shape[0], 'masks found')
 print(masks['ImageId'].value_counts().shape[0])
 masks.head()
masks['ships'] = masks['EncodedPixels'].map(lambda c_row: 1 if isinstance(c_row, str)else 0)
 unique_img_ids = masks.groupby('ImageId' ).agg({'ships': 'sum'} ).reset_index()
unique_img_ids = unique_img_ids[unique_img_ids['ships']>3]
unique_img_ids['has_ship'] = unique_img_ids['ships'].map(lambda x: 1.0 if x>0 else 0.0)
 unique_img_ids['has_ship_vec'] = unique_img_ids['has_ship'].map(lambda x: [x])
 unique_img_ids['file_size_kb'] = unique_img_ids['ImageId'].map(lambda c_img_id:
os.stat(os.path.join(train_image_dir,
c_img_id)).st_size/1024)
 unique_img_ids = unique_img_ids[unique_img_ids['file_size_kb'] > 50]
unique_img_ids['file_size_kb'].hist()
masks.drop(['ships'], axis=1, inplace=True)
 unique_img_ids.sample(7)
 train_ids, valid_ids = train_test_split(unique_img_ids,
test_size = 0.3,
stratify = unique_img_ids['ships'])
 train_df = pd.merge(masks, train_ids)
 valid_df = pd.merge(masks, valid_ids)
 print(train_df.shape[0], 'training masks')
 print(valid_df.shape[0], 'validation masks' )<categorify>
def preprocess(image, target_or_rec_id):
image = tf.image.grayscale_to_rgb(image)
 image = tf.image.resize(image, [params.height,params.width])
 image = tf.image.per_image_standardization(image)
 return image , target_or_rec_id
def _cut_wav(sample, random_shift):
wav, _ = tf.audio.decode_wav(sample['wav'], desired_channels=1)
 tmin = tf.cast(sample['tmin'], tf.float32)
 fmin = tf.cast(sample['fmin'], tf.float32)
 tmax = tf.cast(sample['tmax'], tf.float32)
 fmax = tf.cast(sample['fmax'], tf.float32)
 tmax_s = tmax * tf.cast(params.sample_rate, tf.float32)
 tmin_s = tmin * tf.cast(params.sample_rate, tf.float32)
 cut_s = tf.cast(CUT * params.sample_rate, tf.float32)
 tsize_s = tmax_s - tmin_s
center_s =(tmax_s + tmin_s)/ 2
if random_shift:
shift_limit = tf.math.abs(cut_s - tsize_s)
 shift_s = tf.cast(tf.random.uniform([],-shift_limit/2,shift_limit/2),tf.float32)
 center_s += shift_s
all_s = tf.cast(60 * params.sample_rate, tf.float32)
 cut_min = tf.maximum(0., center_s - cut_s / 2)
 cut_min = tf.minimum(cut_min, all_s - cut_s)
 cut_min = tf.cast(cut_min, tf.int32)
 cut_max = cut_min + CUT * params.sample_rate
wav = tf.squeeze(wav[cut_min : cut_max])
 return wav
def read_labeled_tfrecord(sample, random_shift = False):
target = tf.cast(sample['target'],tf.float32)
 target = tf.squeeze(tf.one_hot([target,], depth = params.num_classes), axis = 0)
 wav = _cut_wav(sample, random_shift)
 return wav, target
def read_labeled_tfrecord_fp(sample, random_shift = False):
target = tf.cast(sample['target'],tf.float32)
 target = tf.squeeze(tf.one_hot([target,], depth = params.num_classes), axis = 0)
 target = target - 1
wav = _cut_wav(sample, random_shift)
 return wav, target
def read_unlabeled_tfrecord(example):
feature_description = {
'recording_id': tf.io.FixedLenFeature([], tf.string),
'audio_wav': tf.io.FixedLenFeature([], tf.string),
}
sample = tf.io.parse_single_example(example, feature_description)
 wav, _ = tf.audio.decode_wav(sample['audio_wav'], desired_channels=1)
 recording_id = tf.reshape(tf.cast(sample['recording_id'] , tf.string), [1])
 def _cut_audio(i):
_sample = {
'audio_wav': tf.reshape(wav[i*params.sample_rate*STRIDE:i*params.sample_rate*STRIDE + params.sample_rate*CUT], [params.sample_rate*CUT]),
'recording_id': sample['recording_id']
}
return _sample
return tf.map_fn(_cut_audio, tf.range(( 60 - CUT)//STRIDE + 1), dtype={
'audio_wav': tf.float32,
'recording_id': tf.string
} )
def masks_as_color(in_mask_list):
all_masks = np.zeros(( 768, 768), dtype = np.float)
 scale = lambda x:(len(in_mask_list)+x+1)/(len(in_mask_list)*2)
 for i,mask in enumerate(in_mask_list):
if isinstance(mask, str):
all_masks[:,:] += scale(i)* rle_decode(mask)
 return all_masks
def raw_prediction(img, path=test_image_dir):
c_img = imread(os.path.join(path, c_img_name))
c_img = np.expand_dims(c_img, 0)/255.0
cur_seg = fullres_model.predict(c_img)[0]
return cur_seg, c_img[0]
def smooth(cur_seg):
return binary_opening(cur_seg>0.57777778, np.expand_dims(disk(2), -1))
def predict(img, path=test_image_dir):
cur_seg, c_img = raw_prediction(img, path=path)
 return smooth(cur_seg), c_img 
<load_pretrained>
def filter_tp_tfrecord(sample):
return tf.cast(sample['is_tp'], tf.bool )
fullres_model = models.load_model('.. /input/attention-train3/seg_model.h5', compile=False )<create_dataframe>
def cutmix(image, label, PROBABILITY = 1.0):
DIM = IMAGE_SIZE[0]
DIM1 = IMAGE_SIZE[1]
imgs = []; labs = []
for j in range(AUG_BATCH):
P = tf.cast(tf.random.uniform([],0,1)<=PROBABILITY, tf.int32)
 k = tf.cast(tf.random.uniform([],0,AUG_BATCH),tf.int32)
 x = tf.cast(tf.random.uniform([],0,DIM),tf.int32)
 y = tf.cast(tf.random.uniform([],0,DIM1),tf.int32)
 b = tf.random.uniform([],0,1)
 WIDTH = tf.cast(DIM * tf.math.sqrt(1-b),tf.int32)* P
HEIGHT = tf.cast(DIM1 * tf.math.sqrt(1-b),tf.int32)* P
ya = tf.math.maximum(0,y-HEIGHT//2)
 yb = tf.math.minimum(DIM1,y+HEIGHT//2)
 xa = tf.math.maximum(0,x-WIDTH//2)
 xb = tf.math.minimum(DIM,x+WIDTH//2)
 one = image[j,ya:yb,0:xa,:]
two = image[k,ya:yb,xa:xb,:]
three = image[j,ya:yb,xb:DIM1,:]
middle = tf.concat([one,two,three],axis=1)
 img = tf.concat([image[j,0:ya,:,:],middle,image[j,yb:DIM,:,:]],axis=0)
 imgs.append(img)
 a = tf.cast(WIDTH*WIDTH/DIM/DIM1,tf.float32)
 if len(label.shape)==1:
lab1 = tf.one_hot(label[j],CLASSES)
 lab2 = tf.one_hot(label[k],CLASSES)
 else:
lab1 = label[j,]
lab2 = label[k,]
labs.append(( 1-a)*lab1 + a*lab2)
 image2 = tf.reshape(tf.stack(imgs),(AUG_BATCH,DIM, DIM1,3))
label2 = tf.reshape(tf.stack(labs),(AUG_BATCH,CLASSES))
return image2,label2
temptest = []
def get_data(sz,bs = 1):
tfms = tfms_from_model(arch, sz, crop_type=CropType.NO, tfm_y=TfmType.CLASS)
 tr_names = tr_n if(len(tr_n_cut)%bs == 0)else tr_n[:-(len(tr_n_cut)%bs)]
ds = ImageData.get_ds(pdFilesDataset,(tr_names,TRAIN), 
(val_n_cut,TRAIN), tfms ,test=(temptest,TEST))
md = ImageData(PATH, ds, bs, num_workers=nw, classes=None)
 return md<load_from_csv>
def mixup(image, label, PROBABILITY = 1.0):
DIM = IMAGE_SIZE[0]
DIM1 = IMAGE_SIZE[1]
imgs = []; labs = []
for j in range(AUG_BATCH):
P = tf.cast(tf.random.uniform([],0,1)<=PROBABILITY, tf.float32)
 k = tf.cast(tf.random.uniform([],0,AUG_BATCH),tf.int32)
 a = tf.random.uniform([],0,1)*P
img1 = image[j,]
img2 = image[k,]
imgs.append(( 1-a)*img1 + a*img2)
 if len(label.shape)==1:
lab1 = tf.one_hot(label[j],CLASSES)
 lab2 = tf.one_hot(label[k],CLASSES)
 else:
lab1 = label[j,]
lab2 = label[k,]
labs.append(( 1-a)*lab1 + a*lab2)
 image2 = tf.reshape(tf.stack(imgs),(AUG_BATCH,DIM, DIM1,3))
label2 = tf.reshape(tf.stack(labs),(AUG_BATCH,CLASSES))
return image2,label2
have_ship= pd.read_csv(".. /input/fine-tuning-resnet34-ship-redo/ship_detection.csv" )<define_variables>
def transform(image,label):
DIM = IMAGE_SIZE[0]
DIM1 = IMAGE_SIZE[1]
SWITCH = 0.5
CUTMIX_PROB = 0.666
MIXUP_PROB = 0.666
image2, label2 = cutmix(image, label, CUTMIX_PROB)
 image3, label3 = mixup(image, label, MIXUP_PROB)
 imgs = []; labs = []
for j in range(AUG_BATCH):
P = tf.cast(tf.random.uniform([],0,1)<=SWITCH, tf.float32)
 imgs.append(P*image2[j,]+(1-P)*image3[j,])
 labs.append(P*label2[j,]+(1-P)*label3[j,])
 image4 = tf.reshape(tf.stack(imgs),(AUG_BATCH,DIM,DIM1,3))
label4 = tf.reshape(tf.stack(labs),(AUG_BATCH,CLASSES))
return image4,label4
ship_list_dict_64 = []
ship_list_dict_73 = []
ship_list_dict_73_more = []
ship_list_dict_82 = []
ship_list_dict_82_more = []
ship_list_dict_en_64 = []
for name in test_names_nothing:
ship_list_dict_64.append({'ImageId':name,'EncodedPixels':None})
 ship_list_dict_73.append({'ImageId':name,'EncodedPixels':None})
 ship_list_dict_73_more.append({'ImageId':name,'EncodedPixels':None})
 ship_list_dict_82.append({'ImageId':name,'EncodedPixels':None})
 ship_list_dict_82_more.append({'ImageId':name,'EncodedPixels':None})
 len(ship_list_dict_64),len(ship_list_dict_73)
<find_best_model_class>
def load_dataset(filenames, labeled = True, ordered = False , training = True):
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads = AUTO)
 dataset = dataset.map(read_original_tfrecord, num_parallel_calls = AUTO)
 if training:
dataset_tp = dataset.filter(filter_tp_tfrecord)
 dataset_fp = dataset.filter(lambda x: not filter_tp_tfrecord(x))
dataset_fp = dataset_fp.map(partial(read_labeled_tfrecord_fp, random_shift=False), num_parallel_calls = AUTO)
 dataset_tp = dataset_tp.map(partial(read_labeled_tfrecord, random_shift=False), num_parallel_calls = AUTO)
 dataset_tp = dataset_tp.map(waveform_to_log_mel_spectrogram , num_parallel_calls = AUTO)
 dataset_tp = dataset_tp.map(apply_augmentation, num_parallel_calls = AUTO)
 dataset_tp = dataset_tp.map(preprocess, num_parallel_calls = AUTO)
 dataset_fp = dataset_fp.map(waveform_to_log_mel_spectrogram , num_parallel_calls = AUTO)
 dataset_fp = dataset_fp.map(apply_augmentation, num_parallel_calls = AUTO)
 dataset_fp = dataset_fp.map(preprocess, num_parallel_calls = AUTO)
 dataset_tp = dataset_tp.batch(GLOBAL_BATCH_SIZE, drop_remainder = True)
 dataset_tp = dataset_tp.map(transform, num_parallel_calls=AUTO)
 dataset_tp = dataset_tp.unbatch()
dataset_fp = dataset_fp.shuffle(256)
 dataset = dataset_tp.concatenate(dataset_fp)
 dataset = dataset.shuffle(2048 ).repeat()
dataset = dataset.batch(GLOBAL_BATCH_SIZE)
 else:
dataset_tp = dataset.filter(filter_tp_tfrecord)
 dataset = dataset_tp.map(read_labeled_tfrecord , num_parallel_calls = AUTO)
 dataset = dataset.map(waveform_to_log_mel_spectrogram , num_parallel_calls = AUTO)
 dataset = dataset.batch(VAL_BATCH_SIZE ).cache()
dataset = dataset.map(preprocess, num_parallel_calls = AUTO)
 dataset = dataset.prefetch(AUTO)
 return dataset
for c_img_name in tqdm_notebook(test_paths):
if c_img_name in test_names:
c_path = os.path.join(test_image_dir, c_img_name)
 c_img = imread(c_path)
 first_img = np.expand_dims(c_img, 0)/255.0
first_seg = fullres_model.predict(first_img)[0]
temptest = [c_img_name]
md = get_data(768,1)
 learn.set_data(md)
 first_seg2 = model_pred_aug(learn, md.test_dl, trms_dihedral)
 ensemble6_4 = first_seg2*0.6 + first_seg[:, :, 0]*0.4
ensemble7_3 = first_seg2*0.7 + first_seg[:, :, 0]*0.3
ensemble7_3_more = ensemble7_3 * 1.05
ensemble6_4_masks = split_mask(ensemble6_4)
 ensemble7_3_masks = split_mask(ensemble7_3)
 ensemble7_3_more_masks = split_mask(ensemble7_3_more)
 if(len(ensemble6_4_masks)== 0):
ship_list_dict_64.append({'ImageId':c_img_name,'EncodedPixels':np.nan})
 for mask in ensemble6_4_masks:
ship_list_dict_64.append({'ImageId':c_img_name,'EncodedPixels':mask})
 if(len(ensemble7_3_masks)== 0):
ship_list_dict_73.append({'ImageId':c_img_name,'EncodedPixels':np.nan})
 for mask in ensemble7_3_masks:
ship_list_dict_73.append({'ImageId':c_img_name,'EncodedPixels':mask})
 ensemble6_4 = None
ensemble7_3 = None
ensemble6_4_masks = None
ensemble7_3_masks = None
gc.collect()
ensemble8_2 = first_seg2*0.8 + first_seg[:, :, 0]*0.2
ensemble8_2_more = ensemble8_2*1.05
ensemble8_2_masks = split_mask(ensemble8_2)
 ensemble8_2_more_masks = split_mask(ensemble8_2_more)
 if(len(ensemble8_2_masks)== 0):
ship_list_dict_82.append({'ImageId':c_img_name,'EncodedPixels':np.nan})
 for mask in ensemble8_2_masks:
ship_list_dict_82.append({'ImageId':c_img_name,'EncodedPixels':mask})
 if(len(ensemble8_2_more_masks)== 0):
ship_list_dict_82_more.append({'ImageId':c_img_name,'EncodedPixels':np.nan})
 for mask in ensemble8_2_more_masks:
ship_list_dict_82_more.append({'ImageId':c_img_name,'EncodedPixels':mask})
 ensemble8_2 = None
ensemble8_2_more = None
ensemble8_2_masks = None
ensemble8_2_more_masks = None
gc.collect()
len(ship_list_dict_64),len(ship_list_dict_73)
<save_to_csv>
TP_FILENAMES = FILENAMES[:8]
FP_FILENAMES = FILENAMES[8:]
submission_64 = pd.DataFrame(ship_list_dict_64)[['ImageId', 'EncodedPixels']]
submission_64.to_csv('submission_64.csv', index=False)
 submission_64.sample(3 )<save_to_csv>
def get_dataset_(filenames, fp_filenames = None, training = True):
if training:
dataset = load_dataset(filenames , training = True)
 dataset = dataset.shuffle(256)
 dataset = dataset.batch(GLOBAL_BATCH_SIZE, drop_remainder = True)
 dataset = dataset.map(transform, num_parallel_calls=AUTO)
 dataset = dataset.unbatch()
if fp_filenames is not None:
dataset_fp = load_dataset(fp_filenames, training = True, FP=True)
 dataset_fp = dataset_fp.shuffle(256)
 dataset = dataset.concatenate(dataset_fp)
 dataset = dataset.shuffle(2048 ).repeat()
dataset = dataset.batch(GLOBAL_BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 else:
dataset = load_dataset(filenames , training = False)
 dataset = dataset.batch(VAL_BATCH_SIZE ).cache()
dataset = dataset.prefetch(AUTO)
 return dataset
submission_73 = pd.DataFrame(ship_list_dict_73)[['ImageId', 'EncodedPixels']]
submission_73.to_csv('submission_73.csv', index=False)
 submission_73.sample(3 )<save_to_csv>
def _one_sample_positive_class_precisions(example):
y_true, y_pred = example
y_true = tf.reshape(y_true, tf.shape(y_pred))
retrieved_classes = tf.argsort(y_pred, direction='DESCENDING')
 class_rankings = tf.argsort(retrieved_classes)
 retrieved_class_true = tf.gather(y_true, retrieved_classes)
 retrieved_cumulative_hits = tf.math.cumsum(tf.cast(retrieved_class_true, tf.float32))
idx = tf.where(y_true)[:, 0]
i = tf.boolean_mask(class_rankings, y_true)
 r = tf.gather(retrieved_cumulative_hits, i)
 c = 1 + tf.cast(i, tf.float32)
 precisions = r / c
dense = tf.scatter_nd(idx[:, None], precisions, [y_pred.shape[0]])
 return dense
class LWLRAP(tf.keras.metrics.Metric):
def __init__(self, num_classes, name='lwlrap'):
super().__init__(name=name)
 self._precisions = self.add_weight(
 name='per_class_cumulative_precision',
shape=[num_classes],
initializer='zeros', 
)
 self._counts = self.add_weight(
 name='per_class_cumulative_count',
shape=[num_classes],
initializer='zeros', 
)
 def update_state(self, y_true, y_pred, sample_weight=None):
precisions = tf.map_fn(
 fn=_one_sample_positive_class_precisions,
elems=(y_true, y_pred),
dtype=(tf.float32), 
)
 increments = tf.cast(precisions > 0, tf.float32)
 total_increments = tf.reduce_sum(increments, axis=0)
 total_precisions = tf.reduce_sum(precisions, axis=0)
 self._precisions.assign_add(total_precisions)
 self._counts.assign_add(total_increments)
 def result(self):
per_class_lwlrap = self._precisions / tf.maximum(self._counts, 1.0)
 per_class_weight = self._counts / tf.reduce_sum(self._counts)
 overall_lwlrap = tf.reduce_sum(per_class_lwlrap * per_class_weight)
 return overall_lwlrap
def reset_states(self):
self._precisions.assign(self._precisions * 0)
 self._counts.assign(self._counts * 0 )
submission_73_more = pd.DataFrame(ship_list_dict_73_more)[['ImageId', 'EncodedPixels']]
submission_73_more.to_csv('submission_73_more.csv', index=False)
 submission_73_more.sample(3 )<save_to_csv>
def cosine_decay_with_warmup(global_step,
learning_rate_base,
total_steps,
warmup_learning_rate=0.0,
warmup_steps= 0,
hold_base_rate_steps=0):
if total_steps < warmup_steps:
raise ValueError('total_steps must be larger or equal to '
'warmup_steps.')
 learning_rate = 0.5 * learning_rate_base *(1 + tf.cos(
 np.pi * 
(tf.cast(global_step, tf.float32)- warmup_steps - hold_base_rate_steps 
)/ float(total_steps - warmup_steps - hold_base_rate_steps)))
 if hold_base_rate_steps > 0:
learning_rate = tf.where(
 global_step > warmup_steps + hold_base_rate_steps,
learning_rate, learning_rate_base)
 if warmup_steps > 0:
if learning_rate_base < warmup_learning_rate:
raise ValueError('learning_rate_base must be larger or equal to '
'warmup_learning_rate.')
 slope =(learning_rate_base - warmup_learning_rate)/ warmup_steps
warmup_rate = slope * tf.cast(global_step,
tf.float32)+ warmup_learning_rate
learning_rate = tf.where(global_step < warmup_steps, warmup_rate,
learning_rate)
 return tf.where(global_step > total_steps, 0.0, learning_rate,
name='learning_rate')
 rng = [i for i in range(int(EPOCHS * STEPS_PER_EPOCH)) ]
WARMUP_STEPS = int(WARMUP_EPOCHS * STEPS_PER_EPOCH)
 y = [cosine_decay_with_warmup(x , LEARNING_RATE, len(rng), 1e-5, WARMUP_STEPS)for x in rng]
sns.set(style='whitegrid')
 fig, ax = plt.subplots(figsize=(20, 6))
plt.plot(rng, y )
submission_82 = pd.DataFrame(ship_list_dict_82)[['ImageId', 'EncodedPixels']]
submission_82.to_csv('submission_82.csv', index=False)
 submission_82.sample(3 )<save_to_csv>
class WarmUpCosineDecayScheduler(tf.keras.callbacks.Callback):
def __init__(self,
learning_rate_base,
total_steps,
global_step_init=0,
warmup_learning_rate=0.0,
warmup_steps=0,
hold_base_rate_steps=0,
verbose=0):
super(WarmUpCosineDecayScheduler, self ).__init__()
self.learning_rate_base = learning_rate_base
self.total_steps = total_steps
self.global_step = global_step_init
self.warmup_learning_rate = warmup_learning_rate
self.warmup_steps = warmup_steps
self.hold_base_rate_steps = hold_base_rate_steps
self.verbose = verbose
self.learning_rates = []
def on_batch_end(self, batch, logs=None):
self.global_step = self.global_step + 1
lr = K.get_value(self.model.optimizer.lr)
 self.learning_rates.append(lr)
 def on_batch_begin(self, batch, logs=None):
lr = cosine_decay_with_warmup(global_step=self.global_step,
learning_rate_base=self.learning_rate_base,
total_steps=self.total_steps,
warmup_learning_rate=self.warmup_learning_rate,
warmup_steps=self.warmup_steps,
hold_base_rate_steps=self.hold_base_rate_steps)
 K.set_value(self.model.optimizer.lr, lr)
 if self.verbose > 0:
print('
Batch %05d: setting learning '
'rate to %s.' %(self.global_step + 1, lr.numpy()))
total_steps = int(EPOCHS * STEPS_PER_EPOCH)
 warmup_steps = int(WARMUP_EPOCHS * STEPS_PER_EPOCH)
 warmup_learning_rate = WARMUP_LEARNING_RATE
submission_82_more = pd.DataFrame(ship_list_dict_82_more)[['ImageId', 'EncodedPixels']]
submission_82_more.to_csv('submission_82_more.csv', index=False)
 submission_82_more.sample(3 )<set_options>
def stat_pool(x):
mean, var = tf.nn.moments(x, axes=[1, 2])
 return K.concatenate([mean, var], axis=1)
 def RFCX_MODEL() :
waveform = Input(shape=(None,None,3), dtype=tf.float32)
 noisy_waveform = GaussianNoise(0.2 )(waveform)
 model = efn.EfficientNetB4(include_top=False, weights='imagenet',)
 model_output = model(noisy_waveform)
 model_output = GlobalAveragePooling2D()(model_output)
 dense = Dropout(params.dropout )(model_output)
 predictions = Dense(params.num_classes, activation = params.classifier_activation )(dense)
 model = Model(
 name='Efficientnet', inputs=waveform,
outputs=[predictions])
 return model
%matplotlib inline<load_from_csv>
def masked_bce(y_true, y_pred):
mask = tf.math.greater(y_true, -0.5)
 loss = tf.keras.losses.binary_crossentropy(tf.boolean_mask(y_true, mask), tf.boolean_mask(y_pred, mask), label_smoothing=0.2)
 return tf.math.reduce_mean(loss )
train = pd.read_csv(DATA / "train.csv")
 test = pd.read_csv(DATA / "test.csv")
 smpl_sub = pd.read_csv(DATA / "sample_submission.csv")
 print("train: {}, test: {}, sample sub: {}".format(
 train.shape, test.shape, smpl_sub.shape 
))<prepare_x_and_y>
def masked_bce_focal(y_true, y_pred):
mask = tf.math.greater(y_true, -0.5)
 loss = tfa.losses.sigmoid_focal_crossentropy(tf.boolean_mask(y_true, mask), tf.boolean_mask(y_pred, mask))
return tf.math.reduce_mean(loss )
class TreeModel:

 def __init__(self, model_type: str):
self.model_type = model_type
self.trn_data = None
self.val_data = None
self.model = None
def train(self,
params: dict,
X_train: pd.DataFrame, y_train: np.ndarray,
X_val: pd.DataFrame, y_val: np.ndarray,
train_weight: tp.Optional[np.ndarray] = None,
val_weight: tp.Optional[np.ndarray] = None,
train_params: dict = {}):
if self.model_type == "lgb":
self.trn_data = lgb.Dataset(X_train, label=y_train, weight=train_weight)
 self.val_data = lgb.Dataset(X_val, label=y_val, weight=val_weight)
 self.model = lgb.train(params=params,
train_set=self.trn_data,
valid_sets=[self.trn_data, self.val_data],
**train_params)
 elif self.model_type == "xgb":
self.trn_data = xgb.DMatrix(X_train, y_train, weight=train_weight)
 self.val_data = xgb.DMatrix(X_val, y_val, weight=val_weight)
 self.model = xgb.train(params=params,
dtrain=self.trn_data,
evals=[(self.trn_data, "train"),(self.val_data, "val")],
**train_params)
 elif self.model_type == "cat":
self.trn_data = Pool(X_train, label=y_train, group_id=[0] * len(X_train))
self.val_data = Pool(X_val, label=y_val, group_id=[0] * len(X_val))
self.model = CatBoost(params)
 self.model.fit(
 self.trn_data, eval_set=[self.val_data], use_best_model=True, **train_params)
 else:
raise NotImplementedError
def predict(self, X: pd.DataFrame):
if self.model_type == "lgb":
return self.model.predict(
 X, num_iteration=self.model.best_iteration)
 elif self.model_type == "xgb":
X_DM = xgb.DMatrix(X)
 return self.model.predict(
 X_DM, ntree_limit=self.model.best_ntree_limit)
 elif self.model_type == "cat":
return self.model.predict(X)
 else:
raise NotImplementedError
@property
def feature_names_(self):
if self.model_type == "lgb":
return self.model.feature_name()
elif self.model_type == "xgb":
return list(self.model.get_score(importance_type="gain" ).keys())
 elif self.model_type == "cat":
return self.model.feature_names_
else:
raise NotImplementedError
@property
def feature_importances_(self):
if self.model_type == "lgb":
return self.model.feature_importance(importance_type="gain")
 elif self.model_type == "xgb":
return list(self.model.get_score(importance_type="gain" ).values())
 elif self.model_type == "cat":
return self.model.feature_importances_
else:
raise NotImplementedError<init_hyperparams>
def get_model() :
with strategy.scope() :
model = RFCX_MODEL()
model.summary()
model.compile(
 optimizer="adam",
loss = masked_bce_focal,
metrics = [LWLRAP(num_classes = params.num_classes),
])
 return model
ID_COL = "id"
FEAT_COLS = [f"cont{i}" for i in range(1, 15)]
TGT_COL = "target"
N_SPLITS = 5
RANDOM_SEED_LIST = [
42,
]
MODEL_PARAMS = {
"lgb": {
"objective": "root_mean_squared_error",
"boosting": "gbdt",
"learning_rate": 0.05,
"seed": RANDOM_SEED_LIST[0],
'max_depth': 7,
'colsample_bytree':.85,
"subsample":.85,
"n_jobs": 2,
},
"xgb": {
"objective": "reg:squarederror",
"learning_rate": 0.01,
"seed": RANDOM_SEED_LIST[0],
"max_depth": 7,
"subsample":.85,
"colsample_bytree":.85,
"n_jobs": 2,
'tree_method': "gpu_hist",
"gpu_id": 0,
},
"cat": {
'loss_function': 'RMSE',
'learning_rate': 0.05,
'max_depth': 7,
'random_state': RANDOM_SEED_LIST[0],
"thread_count": 2,
'num_boost_round': 20000
}
}
TRAIN_PARAMS = {
"lgb": {
"num_boost_round": 20000,
"early_stopping_rounds": 200,
"verbose_eval": 100,
},
"xgb": {
"num_boost_round": 20000,
"early_stopping_rounds": 200,
"verbose_eval": 100,
},
"cat": {'early_stopping_rounds': 200, 'verbose_eval': 100}
}<define_variables>
def get_test_dataset(filenames, training = False):
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads = AUTO)
 dataset = dataset.map(read_unlabeled_tfrecord , num_parallel_calls = AUTO ).unbatch()
dataset = dataset.map(lambda spec : waveform_to_log_mel_spectrogram(spec['audio_wav'], spec['recording_id']), num_parallel_calls = AUTO)
 dataset = dataset.map(preprocess, num_parallel_calls = AUTO)
 return dataset.batch(GLOBAL_BATCH_SIZE*4 ).cache()
use_feat_cols = []
train_feat = train[[ID_COL]].copy()
test_feat = test[[ID_COL]].copy()<concatenate>
test_predict = []
test_data = get_test_dataset(TEST_FILES, training = False)
 test_audio = test_data.map(lambda frames, recording_id: frames)
 model = get_model()
for fold in range(N_FOLDS):
model.load_weights(f'{MODEL_DIR}/RFCX_model_fold {fold}.h5')
 test_predict.append(model.predict(test_audio, verbose = 1))
train_feat = pd.concat([
train_feat, train[FEAT_COLS]], axis=1)
 test_feat = pd.concat([
test_feat, test[FEAT_COLS]], axis=1)
 use_feat_cols.extend(FEAT_COLS )<define_variables>
SUB = pd.read_csv('.. /input/rfcx-species-audio-detection/sample_submission.csv')
 NREC =(60 - CUT)//STRIDE + 1
predict = np.array(test_predict ).reshape(N_FOLDS, len(SUB), NREC, params.num_classes)
 predict = np.mean(np.max(predict ,axis = 2), axis = 0)
 recording_id = test_data.map(lambda frames, recording_id: recording_id ).unbatch()
test_ids = next(iter(recording_id.batch(len(SUB)* NREC)) ).numpy().astype('U' ).reshape(len(SUB), NREC)
 pred_df = pd.DataFrame({ 'recording_id' : test_ids[:, 0],
**{f's{i}' : predict[:, i] for i in range(params.num_classes)} } )
def run_train_and_inference(X, X_test, y, use_model, model_params, train_params, seed_list, n_splits):
oof_pred_arr = np.zeros(len(X))
test_pred_arr = np.zeros(len(X_test))
feature_importances = pd.DataFrame()
score_list = []
for seed in seed_list:
if use_model == "cat":
model_params['random_state'] = seed
else:
model_params["seed"] = seed
kf = KFold(n_splits=n_splits, shuffle=True, random_state=seed)
 tmp_oof_pred = np.zeros(len(X))
tmp_test_pred = np.zeros(len(X_test))
for fold,(trn_idx, val_idx)in enumerate(kf.split(X, y)) :
print("*" * 100)
 print(f"Seed: {seed} - Fold: {fold}")
 X_trn = X.loc[trn_idx].reset_index(drop=True)
 X_val = X.loc[val_idx].reset_index(drop=True)
 y_trn = y[trn_idx]
y_val = y[val_idx]
model = TreeModel(model_type=use_model)
 with timer(prefix="Model training"):
model.train(
 params=model_params, X_train=X_trn, y_train=y_trn,
X_val=X_val, y_val=y_val, train_params=train_params)
 fi_tmp = pd.DataFrame()
fi_tmp["feature"] = model.feature_names_
fi_tmp["importance"] = model.feature_importances_
fi_tmp["fold"] = fold
fi_tmp["seed"] = seed
feature_importances = feature_importances.append(fi_tmp)
 val_pred = model.predict(X_val)
 score = mean_squared_error(y_val, val_pred, squared=False)
 print(f"score: {score:.5f}")
 score_list.append([seed, fold, score])
 tmp_oof_pred[val_idx] = val_pred
tmp_test_pred += model.predict(X_test)
 oof_score = mean_squared_error(y, tmp_oof_pred, squared=False)
 print(f"oof score: {oof_score: 5f}")
 score_list.append([seed, "oof", oof_score])
 oof_pred_arr += tmp_oof_pred
test_pred_arr += tmp_test_pred / n_splits
oof_pred_arr /= len(seed_list)
 test_pred_arr /= len(seed_list)
 oof_score = mean_squared_error(y, oof_pred_arr, squared=False)
 score_list.append(["avg", "oof", oof_score])
 score_df = pd.DataFrame(
 score_list, columns=["seed", "fold", "rmse score"])
 return oof_pred_arr, test_pred_arr, score_df, feature_importances<prepare_x_and_y>
pred_df.sort_values('recording_id', inplace = True)
 pred_df.to_csv('submission.csv', index = False )
<split><EOS> 
MODE = 1
FUDGE = 2.0
for k in range(24):
pred_df.iloc[:,1+k] -= pred_df.iloc[:,1+k].min()
pred_df.iloc[:,1+k] /= pred_df.iloc[:,1+k].max()
def scale(probs, factor):
probs = probs.copy()
idx = np.where(probs!=1)[0]
odds = factor * probs[idx] /(1-probs[idx])
 probs[idx] = odds/(1+odds)
 return probs
d1 = pred_df.iloc[:,1:].mean().values
d2 = np.array([113, 204, 44, 923, 53, 41, 3, 213, 44, 23, 26, 149, 255,
14, 123, 222, 46, 6, 474, 4, 17, 18, 23, 72])/1000.
 for k in range(24):
if MODE==1: d = FUDGE
if MODE==2: d = d1[k]/(1-d1[k])
 if MODE==3: s = d2[k] / d1[k]
else: s =(d2[k]/(1-d2[k])) /d
pred_df.iloc[:,k+1] = scale(pred_df.iloc[:,k+1].values,s)
 pred_df.to_csv('submission_with_pp.csv',index=False )