Kaggle competition: Airbnb New User Bookings
train_best=find_best_train_csv(n_trains,path,3)
 print(train_best)
 BLEND_best=create_csv_list(train_best,path)
 BLEND_best=average_csv(BLEND_best)
 print(BLEND_best)
 BLEND_best.to_csv('submission_best.csv',index=False )<save_to_csv>
train = pd.read_csv('.. /input/airbnb-recruiting-new-user-bookings/train_users_2.csv.zip')
 age_gender = pd.read_csv('.. /input/airbnb-recruiting-new-user-bookings/age_gender_bkts.csv.zip')
 countries_df = pd.read_csv('.. /input/airbnb-recruiting-new-user-bookings/countries.csv.zip')
 session_df = pd.read_csv('.. /input/airbnb-recruiting-new-user-bookings/sessions.csv.zip')
 test = pd.read_csv('.. /input/airbnb-recruiting-new-user-bookings/test_users.csv.zip')
 print('train shape: ', train.shape)
 print('test shape: ', test.shape)
 print('age_gender_bkts: ', age_gender.shape)
 print('countries: ', countries_df.shape)
 print('session: ', session_df.shape )
train_worst=find_worst_train_csv(n_trains,path,2)
 print(train_worst)
 BLEND_worst=create_csv_list(train_worst,path)
 BLEND_worst=average_csv(BLEND_worst)
 print(BLEND_worst)
 BLEND_worst.to_csv('submission_worst.csv',index=False )<save_to_csv>
def glimpse(df, maxvals=10, maxlen=110):
print('Shape: ', df.shape)
 def pad(y):
max_len = max([len(x)for x in y])
 return [x.ljust(max_len)for x in y]
toprnt = pad(df.columns.tolist())
 toprnt = pad([toprnt[i] + ' ' + str(df.iloc[:,i].dtype)for i in range(df.shape[1])])
 num_nas = [df.iloc[:,i].isnull().sum() for i in range(df.shape[1])]
num_nas_ratio = [int(round(x*100/df.shape[0])) for x in num_nas]
num_nas_str = [str(x)+ '(' + str(y)+ '%)' for x,y in zip(num_nas, num_nas_ratio)]
max_len = max([len(x)for x in num_nas_str])
 num_nas_str = [x.rjust(max_len)for x in num_nas_str]
toprnt = [x + ' ' + y + ' NAs' for x,y in zip(toprnt, num_nas_str)]
toprnt = [x + ' : ' for x in toprnt]
toprnt = [toprnt[i] + ', '.join([str(y)for y in df.iloc[:min([maxvals,df.shape[0]]), i]])for i in range(df.shape[1])]
toprnt = [x[:min(maxlen, len(x)) ] for x in toprnt]
for x in toprnt:
print(x )
train_data=read_train_csv(n_trains,path)
 print(train_data)
 BLEND_all=create_csv_list(train_data.nlargest(5, ['best_score']),path)
 BLEND_all=average_csv(BLEND_all)
 print(BLEND_all)
 BLEND_all.to_csv('submission_best_5.csv',index=False )<define_variables>
train.first_affiliate_tracked.value_counts()
save_to_disk = 0<normalization>
train.affiliate_provider.value_counts()
def horizontal_flip(img):
horizontal_flip_img = img[:, ::-1]
return addChannels(horizontal_flip_img)
 def vertical_flip(img):
vertical_flip_img = img[::-1, :]
return addChannels(vertical_flip_img)
 def addNoisy(img):
noise_img = util.random_noise(img)
 return addChannels(noise_img)
 def contrast_stretching(img):
contrast_img = exposure.rescale_intensity(img)
 return addChannels(contrast_img)
 def randomGaussian(img):
gaussian_img = gaussian(img)
 return addChannels(gaussian_img)
 def grayScale(img):
gray_img = rgb2gray(img)
 return addChannels(gray_img)
 def randomGamma(img):
img_gamma = exposure.adjust_gamma(img)
 return addChannels(img_gamma)
 def addChannels(img):
return np.stack(( img, img, img))
def spec_to_image(spec):
spec = resize(spec,(224, 400))
eps=1e-6
mean = spec.mean()
std = spec.std()
spec_norm =(spec - mean)/(std + eps)
 spec_min, spec_max = spec_norm.min() , spec_norm.max()
spec_scaled = 255 *(spec_norm - spec_min)/(spec_max - spec_min)
 spec_scaled = spec_scaled.astype(np.uint8)
 spec_scaled = np.asarray(spec_scaled)
 return spec_scaled<define_variables>
train[train.duplicated() ]
class AudioData(Dataset):
def __init__(self, X, y, data_type):
self.data = []
self.labels = []
for i in range(0, len(X)) :
recording_id = X[i]
label = int(y[i])
 mel_spec = audio_data[recording_id]['original']
self.data.append(mel_spec)
 self.labels.append(label)
 if data_type == "train":
for mel_spec in audio_data[recording_id]['augmentation']:
self.data.append(mel_spec)
 self.labels.append(label)
 def __len__(self):
return len(self.data)
 def __getitem__(self, idx):
return self.data[idx], self.labels[idx]<import_modules>
train['gender'].value_counts()
from torchvision.models import resnet50
import torch
import torch.nn as nn
import copy<init_hyperparams>
train['age'] = train['age'].apply(lambda x: 122 if x > 122 else x)
 train['age'] = train['age'].apply(lambda x: 18 if x < 18 else x )
class Adas(Optimizer):
r
def __init__(self, params,
lr = 0.001, lr2 =.005, lr3 =.0005,
beta_1 = 0.999, beta_2 = 0.999, beta_3 = 0.9999,
epsilon = 1e-8, **kwargs):
if not 0.0 <= lr:
raise ValueError("Invalid lr: {}".format(lr))
if not 0.0 <= lr2:
raise ValueError("Invalid lr2: {}".format(lr))
if not 0.0 <= lr3:
raise ValueError("Invalid lr3: {}".format(lr))
if not 0.0 <= epsilon:
raise ValueError("Invalid epsilon value: {}".format(eps))
if not 0.0 <= beta_1 < 1.0:
raise ValueError("Invalid beta_1 parameter: {}".format(betas[0]))
if not 0.0 <= beta_2 < 1.0:
raise ValueError("Invalid beta_2 parameter: {}".format(betas[1]))
if not 0.0 <= beta_3 < 1.0:
raise ValueError("Invalid beta_3 parameter: {}".format(betas[2]))
defaults = dict(lr=lr, lr2=lr2, lr3=lr3, beta_1=beta_1, beta_2=beta_2, beta_3=beta_3, epsilon=epsilon)
 self._varn = None
self._is_create_slots = None
self._curr_var = None
self._lr = lr
self._lr2 = lr2
self._lr3 = lr3
self._beta_1 = beta_1
self._beta_2 = beta_2
self._beta_3 = beta_3
self._epsilon = epsilon
super(Adas, self ).__init__(params, defaults)
 def __setstate__(self, state):
super(Adas, self ).__setstate__(state)
 @torch.no_grad()
def _add(self,x,y):
x.add_(y)
 return x
@torch.no_grad()
def _derivatives_normalizer(self,derivative,beta):
steps = self._make_variable(0,() ,derivative.dtype)
 self._add(steps,1)
 factor =(1.-(self._beta_1 ** steps)).sqrt()
m = self._make_variable(0,derivative.shape,derivative.dtype)
 moments = self._make_variable(0,derivative.shape,derivative.dtype)
 m.mul_(self._beta_1 ).add_(( 1 - self._beta_1)* derivative * derivative)
 np_t = derivative * factor /(m.sqrt() + self._epsilon)
 return(moments,np_t,lambda: moments.mul_(beta ).add_(( 1 - beta)* np_t))
def _make_variable(self,value,shape,dtype):
self._varn += 1
name = 'unnamed_variable' + str(self._varn)
 if self._is_create_slots:
self.state[self._curr_var][name] = torch.full(size=shape,fill_value=value,dtype=dtype,device=self._curr_var.device)
 return self.state[self._curr_var][name]
@torch.no_grad()
def _get_updates_universal_impl(self, grad, param):
lr = self._make_variable(value = self._lr,shape=param.shape[1:], dtype=param.dtype)
 moment, deriv, f = self._derivatives_normalizer(grad,self._beta_3)
 param.add_(- torch.unsqueeze(lr,0)* deriv)
 lr_deriv = torch.sum(moment * grad,0)
 f()
master_lr = self._make_variable(self._lr2,() ,dtype=torch.float32)
 m2,d2, f = self._derivatives_normalizer(lr_deriv,self._beta_2)
 self._add(lr,master_lr * lr * d2)
 master_lr_deriv2 = torch.sum(m2 * lr_deriv)
 f()
m3,d3,f = self._derivatives_normalizer(master_lr_deriv2,0.)
self._add(master_lr,self._lr3 * master_lr * d3)
 f()
@torch.no_grad()
def _get_updates_universal(self, param, grad, is_create_slots):
self._curr_var = param
self._is_create_slots = is_create_slots
self._varn = 0
return self._get_updates_universal_impl(grad,self._curr_var.data)
 @torch.no_grad()
def step(self, closure=None):

 loss = None
if closure is not None:
with torch.enable_grad() :
loss = closure()
for group in self.param_groups:
for p in group['params']:
if p.grad is None:
continue
grad = p.grad.data
if grad.is_sparse:
raise RuntimeError('Adas does not support sparse gradients')
 self._get_updates_universal(p,grad,len(self.state[p])== 0)
 return loss<set_options>
test['age'] = test['age'].apply(lambda x: 122 if x > 122 else x)
 test['age'] = test['age'].apply(lambda x: 18 if x < 18 else x )
num_birds = 24
if torch.cuda.is_available() :
device=torch.device('cuda:0')
 else:
device=torch.device('cpu' )<train_model>
train['signup_flow'].value_counts()
learning_rate = 2e-4
epochs = 20
loss_fn = nn.CrossEntropyLoss()
def train(model, loss_fn, train_loader, valid_loader, epochs, optimizer):
best_model_wts = copy.deepcopy(model.state_dict())
 best_acc = 0.0
train_losses = []
valid_losses = []
for epoch in tqdm(range(1,epochs+1)) :
model.train()
batch_losses=[]
for i, data in enumerate(train_loader):
x, y = data
optimizer.zero_grad()
x = x.to(device, dtype=torch.float32)
 y = y.to(device, dtype=torch.long)
 y_hat = model(x)
 loss = loss_fn(y_hat, y)
 loss.backward()
batch_losses.append(loss.item())
 optimizer.step()
train_losses.append(batch_losses)
 print(f'Epoch - {epoch} Train-Loss : {np.mean(train_losses[-1])}')
 model.eval()
batch_losses=[]
trace_y = []
trace_yhat = []
for i, data in enumerate(valid_loader):
x, y = data
x = x.to(device, dtype=torch.float32)
 y = y.to(device, dtype=torch.long)
 y_hat = model(x)
 loss = loss_fn(y_hat, y)
 trace_y.append(y.cpu().detach().numpy())
 trace_yhat.append(y_hat.cpu().detach().numpy())
 batch_losses.append(loss.item())
 valid_losses.append(batch_losses)
 trace_y = np.concatenate(trace_y)
 trace_yhat = np.concatenate(trace_yhat)
 accuracy = np.mean(trace_yhat.argmax(axis=1)==trace_y)
 print(f'Epoch - {epoch} Valid-Loss : {np.mean(valid_losses[-1])} Valid-Accuracy : {accuracy}')
 if accuracy > best_acc:
best_acc = accuracy
best_model_wts = copy.deepcopy(model.state_dict())
 model.load_state_dict(best_model_wts)
 return model<load_from_csv>
train['signup_method'].value_counts()
fft = 2048
hop = 512
sr = 48000
length = 10 * sr
with open('/kaggle/input/rfcx-species-audio-detection/train_tp.csv')as f:
reader = csv.reader(f)
 next(reader, None)
 data = list(reader)
 fmin = 24000
fmax = 0
for i in range(0, len(data)) :
if fmin > float(data[i][4]):
fmin = float(data[i][4])
 if fmax < float(data[i][6]):
fmax = float(data[i][6])
 fmin = int(fmin * 0.9)
 fmax = int(fmax * 1.1)
 print('Minimum frequency: ' + str(fmin)+ ', maximum frequency: ' + str(fmax))
label_list = []
data_list = []
audio_data = {}
for i in range(0, len(data)) :
recording_id = data[i][0]
species_id = data[i][1]
data_list.append(recording_id)
 label_list.append(species_id)
 audio_data[recording_id] = {}
wav, sr = librosa.load('/kaggle/input/rfcx-species-audio-detection/train/' + recording_id + '.flac', sr=None)
 t_min = float(data[i][3])* sr
t_max = float(data[i][5])* sr
center = np.round(( t_min + t_max)/ 2)
 beginning = center - length / 2
if beginning < 0:
beginning = 0
ending = beginning + length
if ending > len(wav):
ending = len(wav)
 beginning = ending - length
slice = wav[int(beginning):int(ending)]
spec=librosa.feature.melspectrogram(slice, sr=sr,n_fft=fft,hop_length=hop,fmin=fmin,fmax=fmax)
 spec_db=librosa.power_to_db(spec,top_db=80)
 img = spec_to_image(spec_db)
 mel_spec = np.stack(( img, img, img))
audio_data[recording_id]["original"] = mel_spec
audio_data[recording_id]["augmentation"] = []
augmentation_functions = [
addNoisy, contrast_stretching,
randomGaussian, grayScale,
randomGamma, vertical_flip,
horizontal_flip
]
for fun in augmentation_functions:
mel_spec = fun(img)
 audio_data[recording_id]["augmentation"].append(mel_spec )<prepare_x_and_y>
train[train.duplicated() ]
nfold = 5
skf = StratifiedKFold(n_splits=nfold, shuffle=True, random_state=32)
 for fold_id,(train_index, val_index)in enumerate(skf.split(data_list, label_list)) :
print("Fold", fold_id)
 X_train = np.take(data_list, train_index)
 y_train = np.take(label_list, train_index)
 X_val = np.take(data_list, val_index)
 y_val = np.take(label_list, val_index)
 train_data = AudioData(X_train, y_train, "train")
 valid_data = AudioData(X_val, y_val, "valid")
 train_loader = DataLoader(train_data, batch_size=16, shuffle=True)
 valid_loader = DataLoader(valid_data, batch_size=16, shuffle=True)
 resnet_model = resnet50(pretrained=True)
 num_ftrs = resnet_model.fc.in_features
resnet_model.fc = nn.Linear(num_ftrs, num_birds)
 resnet_model = resnet_model.to(device)
 optimizer = Adas(resnet_model.parameters() , lr=learning_rate)
 resnet_model = train(resnet_model, loss_fn, train_loader, valid_loader, epochs, optimizer)
 torch.save(resnet_model.state_dict() , "model"+str(fold_id)+".pt" )<choose_model_class>
session_df['device_type'].value_counts()
members = []
for i in range(nfold):
model = resnet50(pretrained=True)
 num_ftrs = model.fc.in_features
model.fc = nn.Linear(num_ftrs, num_birds)
 model = model.to(device)
 model.load_state_dict(torch.load('/kaggle/working/model'+str(i)+'.pt'))
model.eval()
members.append(model )<load_pretrained>
session_df['action_detail'].value_counts() [:10]
def load_test_file(f):
wav, sr = librosa.load('/kaggle/input/rfcx-species-audio-detection/test/' + f, sr=None)
 segments = len(wav)/ length
segments = int(np.ceil(segments))
mel_array = []
for i in range(0, segments):
if(i + 1)* length > len(wav):
slice = wav[len(wav)- length:len(wav)]
else:
slice = wav[i * length:(i + 1)* length]
spec=librosa.feature.melspectrogram(slice, sr=sr,n_fft=fft,hop_length=hop,fmin=fmin,fmax=fmax)
 spec_db=librosa.power_to_db(spec,top_db=80)
 img = spec_to_image(spec_db)
 mel_spec = np.stack(( img, img, img))
mel_array.append(mel_spec)
 return mel_array<save_to_csv>
view_search_time = session_df[session_df.action_detail == 'view_search_results']
view_search_time
if save_to_disk == 0:
for f in os.listdir('/kaggle/working/'):
os.remove('/kaggle/working/' + f)
 print('Starting prediction loop')
 with open('submission.csv', 'w', newline='')as csvfile:
submission_writer = csv.writer(csvfile, delimiter=',')
 submission_writer.writerow(['recording_id','s0','s1','s2','s3','s4','s5','s6','s7','s8','s9','s10','s11',
's12','s13','s14','s15','s16','s17','s18','s19','s20','s21','s22','s23'])
 test_files = os.listdir('/kaggle/input/rfcx-species-audio-detection/test/')
 print(len(test_files))
for i in range(0, len(test_files)) :
data = load_test_file(test_files[i])
 data = torch.tensor(data)
 data = data.float()
if torch.cuda.is_available() :
data = data.cuda()
output_list = []
for m in members:
output = m(data)
 maxed_output = torch.max(output, dim=0)[0]
maxed_output = maxed_output.cpu().detach()
output_list.append(maxed_output)
 avg_maxed_output = torch.mean(torch.stack(output_list), dim=0)
 file_id = str.split(test_files[i], '.')[0]
write_array = [file_id]
for out in avg_maxed_output:
write_array.append(out.item())
 submission_writer.writerow(write_array)
 if i % 100 == 0 and i > 0:
print('Predicted for ' + str(i)+ ' of ' + str(len(test_files)+ 1)+ ' files')
 print('Submission generated' )<import_modules>
labels = train['country_destination']
train.drop('country_destination', inplace = True, axis = 1 )
COLAB=False
models_path=''
if not os.path.exists('.. /input/rfcx-species-audio-detection'):
COLAB=True
drive.mount('/content/drive')
 GCS_DS_PATH = 'gs://kds-5c677f76ce55440722b2a474a5492faa70847c05a8f5d722f5a37feb'
models_path='/content/drive/MyDrive/Models/'
else:
GCS_DS_PATH = KaggleDatasets().get_gcs_path('rfcx-species-audio-detection')
 print(GCS_DS_PATH )<load_pretrained>
data = pd.concat(( train, test), axis=0, ignore_index=True)
 data = data.drop(['id', 'date_first_booking'], axis=1 )
if COLAB:
if not os.path.exists('/.kaggle/kaggle.json'):
!mkdir ~/.kaggle
if not os.path.exists('/content/drive/My Drive/Models/kaggle.json'):
files.upload()
!cp kaggle.json ~/.kaggle/
else:
!cp '/content/drive/My Drive/Models/kaggle.json' ~/.kaggle/
!chmod 600 ~/.kaggle/kaggle.json<train_model>
cat_features = ['gender', 'signup_method', 'signup_flow', 'language', 'affiliate_channel',
'affiliate_provider', 'first_affiliate_tracked', 'signup_app', 'first_device_type', 'first_browser']
for f in cat_features:
data_dummy = pd.get_dummies(data[f], prefix=f)
 data.drop([f], axis=1, inplace = True)
 data = pd.concat(( data, data_dummy), axis=1 )
if COLAB:
!pip install -q tensorflow~=2.2.0 tensorflow_gcs_config~=2.2.0
resp = requests.post("http://{}:8475/requestversion/{}".format(os.environ["COLAB_TPU_ADDR"].split(":")[0], tf.__version__))
if resp.status_code != 200:
print("Failed to switch the TPU to TF {}".format(version))<choose_model_class>
from datetime import datetime
from sklearn.preprocessing import LabelEncoder
if COLAB:
print("Tensorflow version " + tf.__version__)
 try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.cluster_spec().as_dict() ['worker'])
 except ValueError:
raise BaseException('ERROR: Not connected to a TPU runtime; please see the previous cell in this notebook for instructions!')
 tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 tpu_strategy = tf.distribute.experimental.TPUStrategy(tpu)
<import_modules>
data['date_account_created'] = pd.to_datetime(data['date_account_created'] )
!pip install image-classifiers
!pip install tensorflow_addons==0.10.0
!pip install soundfile
tf.__version__<import_modules>
data['ac_year'] = data['date_account_created'].dt.year
data['ac_month'] = data['date_account_created'].dt.month
data['ac_day'] = data['date_account_created'].dt.day
from classification_models.keras import Classifiers<define_variables>
data.drop('date_account_created', inplace = True, axis = 1 )
SEED = random.randint(0, 10000)
 def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 seed_everything(SEED )<choose_model_class>
data['timestamp_first_active'] = pd.to_datetime(data['timestamp_first_active'] )
ResNet34, preprocess_input = Classifiers.get('resnet34' )<init_hyperparams>
data['ts_fa_year'] = data['timestamp_first_active'].dt.year
data['ts_fa_month'] = data['timestamp_first_active'].dt.month
data['ts_fa_day'] = data['timestamp_first_active'].dt.day
cfg = {
'parse_params': {
'cut_time': 10,
},
'data_params': {
'sample_time': 6,
'spec_fmax': 24000.0,
'spec_fmin': 40.0,
'spec_mel': 300,
'mel_power': 2,
'img_shape':(300, 670)
 },
'model_params': {
'batchsize_per_tpu': 8,
'iteration_per_epoch': 128,
'epoch': 25,
'arch': ResNet34,
'arch_preprocess': preprocess_input,
'freeze_to': 0,
'loss': {
'fn': tfa.losses.SigmoidFocalCrossEntropy,
'params': {},
},
'optim': {
'fn': tfa.optimizers.RectifiedAdam,
'params': {'lr': 2e-3, 'total_steps': 18*64, 'warmup_proportion': 0.3, 'min_lr': 1e-6},
},
'mixup': True
}
}<choose_model_class>
data.drop('timestamp_first_active', inplace = True, axis = 1 )
if not COLAB:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 print("All devices: ", tf.config.list_logical_devices('TPU'))
tpu_strategy = tf.distribute.experimental.TPUStrategy(tpu )<define_variables>
le = LabelEncoder()
y = le.fit_transform(labels )
AUTOTUNE = tf.data.experimental.AUTOTUNE
TRAIN_TFREC = GCS_DS_PATH + "/tfrecords/train"
TEST_TFREC = GCS_DS_PATH + "/tfrecords/test"<define_variables>
X = data[:train.shape[0]]
X_test = data[train.shape[0]:]
CUT = cfg['parse_params']['cut_time']
SR = 48000
TIME = cfg['data_params']['sample_time']
FMAX = cfg['data_params']['spec_fmax']
FMIN = cfg['data_params']['spec_fmin']
N_MEL = cfg['data_params']['spec_mel']
HEIGHT, WIDTH = cfg['data_params']['img_shape']
CLASS_N = 24<create_dataframe>
xgb = XGBClassifier(use_label_encoder=False)
 xgb.fit(X, y)
raw_dataset = tf.data.TFRecordDataset([TRAIN_TFREC + '/00-148.tfrec'])
 raw_dataset<categorify>
y_pred = xgb.predict_proba(X_test )
feature_description = {
'recording_id': tf.io.FixedLenFeature([], tf.string, default_value=''),
'audio_wav': tf.io.FixedLenFeature([], tf.string, default_value=''),
'label_info': tf.io.FixedLenFeature([], tf.string, default_value=''),
}
parse_dtype = {
'audio_wav': tf.float32,
'recording_id': tf.string,
'species_id': tf.int32,
'songtype_id': tf.int32,
't_min': tf.float32,
'f_min': tf.float32,
't_max': tf.float32,
'f_max':tf.float32,
'is_tp': tf.int32
}
@tf.function
def _parse_function(example_proto):
sample = tf.io.parse_single_example(example_proto, feature_description)
 wav, _ = tf.audio.decode_wav(sample['audio_wav'], desired_channels=1)
 label_info = tf.strings.split(sample['label_info'], sep='"')[1]
labels = tf.strings.split(label_info, sep=';')
 @tf.function
def _cut_audio(label):
items = tf.strings.split(label, sep=',')
 spid = tf.squeeze(tf.strings.to_number(items[0], tf.int32))
soid = tf.squeeze(tf.strings.to_number(items[1], tf.int32))
tmin = tf.squeeze(tf.strings.to_number(items[2]))
fmin = tf.squeeze(tf.strings.to_number(items[3]))
tmax = tf.squeeze(tf.strings.to_number(items[4]))
fmax = tf.squeeze(tf.strings.to_number(items[5]))
tp = tf.squeeze(tf.strings.to_number(items[6], tf.int32))
tmax_s = tmax * tf.cast(SR, tf.float32)
 tmin_s = tmin * tf.cast(SR, tf.float32)
 cut_s = tf.cast(CUT * SR, tf.float32)
 all_s = tf.cast(60 * SR, tf.float32)
 tsize_s = tmax_s - tmin_s
cut_min = tf.cast(
 tf.maximum(0.0,
tf.minimum(tmin_s -(cut_s - tsize_s)/ 2,
tf.minimum(tmax_s +(cut_s - tsize_s)/ 2, all_s)- cut_s)
), tf.int32 
)
 cut_max = cut_min + CUT * SR
_sample = {
'audio_wav': tf.reshape(wav[cut_min:cut_max], [CUT*SR]),
'recording_id': sample['recording_id'],
'species_id': spid,
'songtype_id': soid,
't_min': tmin - tf.cast(cut_min, tf.float32)/tf.cast(SR, tf.float32),
'f_min': fmin,
't_max': tmax - tf.cast(cut_min, tf.float32)/tf.cast(SR, tf.float32),
'f_max': fmax,
'is_tp': tp
}
return _sample
samples = tf.map_fn(_cut_audio, labels, dtype=parse_dtype)
 return samples
parsed_dataset = raw_dataset.map(_parse_function ).unbatch()<feature_engineering>
ids = []
countries = []
test_id = test['id']
@tf.function
def _cut_wav(x):
cut_min = tf.random.uniform([], maxval=(CUT-TIME)*SR, dtype=tf.int32)
 cut_max = cut_min + TIME * SR
cutwave = tf.reshape(x['audio_wav'][cut_min:cut_max], [TIME*SR])
 y = {}
y.update(x)
 y['audio_wav'] = cutwave
y['t_min'] = tf.maximum(0.0, x['t_min'] - tf.cast(cut_min, tf.float32)/ SR)
 y['t_max'] = tf.maximum(0.0, x['t_max'] - tf.cast(cut_min, tf.float32)/ SR)
 return y
@tf.function
def _cut_wav_val(x):
cut_min =(CUT-TIME)*SR // 2
cut_max = cut_min + TIME * SR
cutwave = tf.reshape(x['audio_wav'][cut_min:cut_max], [TIME*SR])
 y = {}
y.update(x)
 y['audio_wav'] = cutwave
y['t_min'] = tf.maximum(0.0, x['t_min'] - tf.cast(cut_min, tf.float32)/ SR)
 y['t_max'] = tf.maximum(0.0, x['t_max'] - tf.cast(cut_min, tf.float32)/ SR)
 return y<feature_engineering>
for i in range(len(test_id)) :
idx = test_id[i]
ids += [idx] * 5
countries += le.inverse_transform(np.argsort(y_pred[i])[::-1])[:5].tolist()
<data_type_conversions><EOS> 
submission = pd.DataFrame(np.column_stack(( ids, countries)) , columns=['id', 'country'])
 submission.to_csv('submission.csv',index=False )