Kaggle competition: Rainforest Connection Species Audio Detection
<SOS> metric: WeightedLabelRankingAveragePrecision Kaggle data source: rainforest-connection-species-audio-detection<import_modules>
tf.__version__
import numpy as np
import pandas as pd<categorify>
cfg = {
'parse_params': {
'cut_time': 10,
},
'data_params': {
'sample_time': 6,
'spec_fmax': 24000.0,
'spec_fmin': 40.0,
'spec_mel': 224,
'mel_power': 2,
'img_shape':(224, 512)
 },
'model_params': {
'batchsize_per_tpu': 64,
'iteration_per_epoch': 64,
'epoch': 18,
'arch': tf.keras.applications.ResNet50,
'arch_preprocess': tf.keras.applications.resnet50.preprocess_input,
'freeze_to': 0,
'loss': {
'fn': tfa.losses.SigmoidFocalCrossEntropy,
'params': {},
},
'optim': {
'fn': tfa.optimizers.RectifiedAdam,
'params': {'lr': 1e-3, 'total_steps': 18*64, 'warmup_proportion': 0.3, 'min_lr': 1e-6},
},
'mixup': False
}
}
def rle2mask(mask_rle, shape=(1600,256)) :

 s = mask_rle.split()
starts, lengths = [np.asarray(x, dtype=int)for x in(s[0:][::2], s[1:][::2])]
starts -= 1
ends = starts + lengths
img = np.zeros(shape[0]*shape[1], dtype=np.uint8)
 for lo, hi in zip(starts, ends):
img[lo:hi] = 1
return img.reshape(shape ).T
def rle_encode_less_memory(img):

 pixels = img.T.flatten()
pixels[0] = 0
pixels[-1] = 0
runs = np.where(pixels[1:] != pixels[:-1])[0] + 2
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs)
 def image_size_dict(img_id, x, y):
image_id = [thing[:-5] for thing in img_id]
x_y = [(x[i], y[i])for i in range(0, len(x)) ]
return dict(zip(image_id, x_y))
def global_shift_mask(maskpred1, y_shift, x_shift):

 if y_shift <0 and x_shift >=0:
maskpred2 = np.pad(maskpred1, [(0,abs(y_shift)) ,(abs(x_shift), 0)], mode='constant', constant_values=0)
 maskpred3 = maskpred2[abs(y_shift):, :maskpred1.shape[1]]
elif y_shift >=0 and x_shift <0:
maskpred2 = np.pad(maskpred1, [(abs(y_shift),0),(0, abs(x_shift)) ], mode='constant', constant_values=0)
 maskpred3 = maskpred2[:maskpred1.shape[0], abs(x_shift):]
elif y_shift >=0 and x_shift >=0:
maskpred2 = np.pad(maskpred1, [(abs(y_shift),0),(abs(x_shift), 0)], mode='constant', constant_values=0)
 maskpred3 = maskpred2[:maskpred1.shape[0], :maskpred1.shape[1]]
elif y_shift < 0 and x_shift < 0:
maskpred2 = np.pad(maskpred1, [(0, abs(y_shift)) ,(0, abs(x_shift)) ], mode='constant', constant_values=0)
 maskpred3 = maskpred2[abs(y_shift):, abs(x_shift):]
return maskpred3<load_from_csv>
strategy = tf.distribute.experimental.TPUStrategy(tpu)
 AUTOTUNE = tf.data.experimental.AUTOTUNE
GCS_DS_PATH = KaggleDatasets().get_gcs_path()
TRAIN_TFREC = GCS_DS_PATH + "/tfrecords/train"
TEST_TFREC = GCS_DS_PATH + "/tfrecords/test"
dfpred = pd.read_csv('.. /input/best-public-hubmap/submission_public_TPU.csv')
 TARGET_ID = 'afa5e8098'
y_shift = -40
x_shift = -24<load_from_csv>
CUT = cfg['parse_params']['cut_time']
SR = 48000
TIME = cfg['data_params']['sample_time']
FMAX = cfg['data_params']['spec_fmax']
FMIN = cfg['data_params']['spec_fmin']
N_MEL = cfg['data_params']['spec_mel']
HEIGHT, WIDTH = cfg['data_params']['img_shape']
CLASS_N = 24
dfinfo = pd.read_csv('.. /input/hubmap-kidney-segmentation/HuBMAP-20-dataset_information.csv')
 size_dict = image_size_dict(dfinfo.image_file, dfinfo.width_pixels, dfinfo.height_pixels)
 mask_shape = size_dict.get(TARGET_ID)
 taridx = dfpred[dfpred['id']==TARGET_ID].index.values[0]<normalization>
raw_dataset = tf.data.TFRecordDataset([TRAIN_TFREC + '/00-148.tfrec'])
 raw_dataset
maskpred = rle2mask(dfpred.iloc[taridx]['predicted'], mask_shape)
 maskpred1 = maskpred.copy()
maskpred1[maskpred1>0]=1<normalization>
feature_description = {
'recording_id': tf.io.FixedLenFeature([], tf.string, default_value=''),
'audio_wav': tf.io.FixedLenFeature([], tf.string, default_value=''),
'label_info': tf.io.FixedLenFeature([], tf.string, default_value=''),
}
parse_dtype = {
'audio_wav': tf.float32,
'recording_id': tf.string,
'species_id': tf.int32,
'songtype_id': tf.int32,
't_min': tf.float32,
'f_min': tf.float32,
't_max': tf.float32,
'f_max':tf.float32,
'is_tp': tf.int32
}
@tf.function
def _parse_function(example_proto):
sample = tf.io.parse_single_example(example_proto, feature_description)
 wav, _ = tf.audio.decode_wav(sample['audio_wav'], desired_channels=1)
 label_info = tf.strings.split(sample['label_info'], sep='"')[1]
labels = tf.strings.split(label_info, sep=';')
 @tf.function
def _cut_audio(label):
items = tf.strings.split(label, sep=',')
 spid = tf.squeeze(tf.strings.to_number(items[0], tf.int32))
soid = tf.squeeze(tf.strings.to_number(items[1], tf.int32))
tmin = tf.squeeze(tf.strings.to_number(items[2]))
fmin = tf.squeeze(tf.strings.to_number(items[3]))
tmax = tf.squeeze(tf.strings.to_number(items[4]))
fmax = tf.squeeze(tf.strings.to_number(items[5]))
tp = tf.squeeze(tf.strings.to_number(items[6], tf.int32))
tmax_s = tmax * tf.cast(SR, tf.float32)
 tmin_s = tmin * tf.cast(SR, tf.float32)
 cut_s = tf.cast(CUT * SR, tf.float32)
 all_s = tf.cast(60 * SR, tf.float32)
 tsize_s = tmax_s - tmin_s
cut_min = tf.cast(
 tf.maximum(0.0,
tf.minimum(tmin_s -(cut_s - tsize_s)/ 2,
tf.minimum(tmax_s +(cut_s - tsize_s)/ 2, all_s)- cut_s)
), tf.int32 
)
 cut_max = cut_min + CUT * SR
_sample = {
'audio_wav': tf.reshape(wav[cut_min:cut_max], [CUT*SR]),
'recording_id': sample['recording_id'],
'species_id': spid,
'songtype_id': soid,
't_min': tmin - tf.cast(cut_min, tf.float32)/tf.cast(SR, tf.float32),
'f_min': fmin,
't_max': tmax - tf.cast(cut_min, tf.float32)/tf.cast(SR, tf.float32),
'f_max': fmax,
'is_tp': tp
}
return _sample
samples = tf.map_fn(_cut_audio, labels, dtype=parse_dtype)
 return samples
parsed_dataset = raw_dataset.map(_parse_function ).unbatch()
mask_shifted = global_shift_mask(maskpred1, y_shift, x_shift )<save_to_csv>
@tf.function
def _cut_wav(x):
cut_min = tf.random.uniform([], maxval=(CUT-TIME)*SR, dtype=tf.int32)
 cut_max = cut_min + TIME * SR
cutwave = tf.reshape(x['audio_wav'][cut_min:cut_max], [TIME*SR])
 y = {}
y.update(x)
 y['audio_wav'] = cutwave
y['t_min'] = tf.maximum(0.0, x['t_min'] - tf.cast(cut_min, tf.float32)/ SR)
 y['t_max'] = tf.maximum(0.0, x['t_max'] - tf.cast(cut_min, tf.float32)/ SR)
 return y
@tf.function
def _cut_wav_val(x):
cut_min =(CUT-TIME)*SR // 2
cut_max = cut_min + TIME * SR
cutwave = tf.reshape(x['audio_wav'][cut_min:cut_max], [TIME*SR])
 y = {}
y.update(x)
 y['audio_wav'] = cutwave
y['t_min'] = tf.maximum(0.0, x['t_min'] - tf.cast(cut_min, tf.float32)/ SR)
 y['t_max'] = tf.maximum(0.0, x['t_max'] - tf.cast(cut_min, tf.float32)/ SR)
 return y
newrle = rle_encode_less_memory(mask_shifted)
 dfpred.at[taridx, 'predicted'] = newrle
dfsample = pd.read_csv('.. /input/hubmap-kidney-segmentation/sample_submission.csv')
 mydict = dict(zip(dfpred['id'], dfpred['predicted']))
dfsample['predicted'] = dfsample['id'].map(mydict ).fillna(dfsample['predicted'])
 dfsample = dfsample.replace(np.nan, '', regex=True)
 dfsample.to_csv('submission.csv',index=False)
<import_modules>
@tf.function
def _filtTP(x):
return x['is_tp'] == 1
import numpy as np
import pandas as pd
from sklearn.linear_model import LogisticRegression<load_from_csv>
def show_wav(sample, ax):
wav = sample["audio_wav"].numpy()
rate = SR
ax.plot(np.arange(len(wav)) / rate, wav)
 ax.set_title(
 sample["recording_id"].numpy().decode()
+("/%d" % sample["species_id"])
 +("TP" if sample["is_tp"] else "FP"))
return Audio(( wav * 2**15 ).astype(np.int16), rate=rate)
 fig, ax = plt.subplots(figsize=(15, 3))
show_wav(next(iter(parsed_dataset)) , ax )
data_dir = '.. /input/'
tourresults = pd.read_csv(data_dir + 'WNCAATourneyCompactResults.csv')
 tourseeds = pd.read_csv(data_dir + 'WNCAATourneySeeds.csv')
 seasonresults = pd.read_csv(data_dir + 'WRegularSeasonCompactResults.csv' )<define_variables>
@tf.function
def _wav_to_spec(x):
mel_power = cfg['data_params']['mel_power']
stfts = tf.signal.stft(x["audio_wav"], frame_length=2048, frame_step=512, fft_length=2048)
 spectrograms = tf.abs(stfts)** mel_power
num_spectrogram_bins = stfts.shape[-1]
lower_edge_hertz, upper_edge_hertz, num_mel_bins = FMIN, FMAX, N_MEL
linear_to_mel_weight_matrix = tf.signal.linear_to_mel_weight_matrix(
 num_mel_bins, num_spectrogram_bins, SR, lower_edge_hertz,
upper_edge_hertz)
 mel_spectrograms = tf.tensordot(
 spectrograms, linear_to_mel_weight_matrix, 1)
 mel_spectrograms.set_shape(spectrograms.shape[:-1].concatenate(
 linear_to_mel_weight_matrix.shape[-1:]))
log_mel_spectrograms = tf.math.log(mel_spectrograms + 1e-6)
 y = {
'audio_spec': tf.transpose(log_mel_spectrograms),
}
y.update(x)
 return y
spec_dataset = parsed_dataset.filter(_filtTP ).map(_cut_wav ).map(_wav_to_spec )
def GetWinsLosses(seasonresults, year, teamID):
Spec_season = seasonresults['Season']==year
teamIDW = seasonresults['WTeamID']==teamID
teamIDL = seasonresults['LTeamID']==teamID
wins = len(seasonresults[Spec_season & teamIDW])
 losses = len(seasonresults[Spec_season & teamIDL])
 return [wins, losses]
def seed_to_int(seed):
s_int = int(seed[1:3])
 return s_int<feature_engineering>
@tf.function
def _create_annot(x):
targ = tf.one_hot(x["species_id"], CLASS_N, on_value=x["is_tp"], off_value=0)
 return {
'input': x["audio_spec"],
'target': tf.cast(targ, tf.float32)
 }
annot_dataset = spec_dataset.map(_create_annot )
year = 2018
df_seeds = tourseeds[tourseeds['Season']==year]
df_tour = tourresults[tourresults['Season']==year]
df_season = seasonresults[seasonresults['Season']==year]
df_seeds['seed_int'] = df_seeds.Seed.apply(seed_to_int)
 df_seeds.drop(labels=['Seed'], inplace=True, axis=1 )<drop_column>
@tf.function
def _preprocess_img(x, training=False):
image = tf.expand_dims(x, axis=-1)
 image = tf.image.resize(image, [HEIGHT, WIDTH])
 image = tf.image.per_image_standardization(image)
 @tf.function
def _specaugment(image):
ERASE_TIME = 50
ERASE_MEL = 16
image = tf.expand_dims(image, axis=0)
 xoff = tf.random.uniform([2], minval=ERASE_TIME//2, maxval=WIDTH-ERASE_TIME//2, dtype=tf.int32)
 xsize = tf.random.uniform([2], minval=ERASE_TIME//2, maxval=ERASE_TIME, dtype=tf.int32)
 yoff = tf.random.uniform([2], minval=ERASE_MEL//2, maxval=HEIGHT-ERASE_MEL//2, dtype=tf.int32)
 ysize = tf.random.uniform([2], minval=ERASE_MEL//2, maxval=ERASE_MEL, dtype=tf.int32)
 image = tfa.image.cutout(image, [HEIGHT, xsize[0]], offset=[HEIGHT//2, xoff[0]])
 image = tfa.image.cutout(image, [HEIGHT, xsize[1]], offset=[HEIGHT//2, xoff[1]])
 image = tfa.image.cutout(image, [ysize[0], WIDTH], offset=[yoff[0], WIDTH//2])
 image = tfa.image.cutout(image, [ysize[1], WIDTH], offset=[yoff[1], WIDTH//2])
 image = tf.squeeze(image, axis=0)
 return image
if training:
gau = tf.keras.layers.GaussianNoise(0.3)
 image = tf.cond(tf.random.uniform([])< 0.5, lambda: gau(image, training=True), lambda: image)
 image = tf.image.random_brightness(image, 0.2)
 image = tf.cond(tf.random.uniform([])< 0.5, lambda: _specaugment(image), lambda: image)
 image =(image - tf.reduce_min(image)) /(tf.reduce_max(image)- tf.reduce_min(image)) * 255.0
image = tf.image.grayscale_to_rgb(image)
 image = cfg['model_params']['arch_preprocess'](image)
 return image
@tf.function
def _preprocess(x):
image = _preprocess_img(x['input'], True)
 return(image, x["target"])
 @tf.function
def _preprocess_val(x):
image = _preprocess_img(x['input'], False)
 return(image, x["target"])
 @tf.function
def _preprocess_test(x):
image = _preprocess_img(x['audio_spec'], False)
 return(image, x["recording_id"] )
seasonresultsNew = seasonresults.drop(['DayNum','WScore','LScore','WLoc','NumOT'],axis=1 )<feature_engineering>
def create_model() :
with strategy.scope() :
backbone = cfg['model_params']['arch'](include_top=False, weights='imagenet')
 if cfg['model_params']['freeze_to'] is None:
for layer in backbone.layers:
layer.trainable = False
else:
for layer in backbone.layers[:cfg['model_params']['freeze_to']]:
layer.trainable = False
model = tf.keras.Sequential([
backbone,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.4),
tf.keras.layers.Dense(1024, activation='relu', kernel_initializer=tf.keras.initializers.he_normal()),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.4),
tf.keras.layers.Dense(CLASS_N, bias_initializer=tf.keras.initializers.Constant(-2.))])
 return model
model = create_model()
model.summary()
Teams = df_seeds.TeamID.sort_values().unique()
n_test_games = int(len(Teams)*(len(Teams)-1)/2)
 df_winseeds = df_seeds.rename(columns={'TeamID':'WTeamID', 'seed_int':'WSeed'})
 df_lossseeds = df_seeds.rename(columns={'TeamID':'LTeamID', 'seed_int':'LSeed'})
 CompleteDF = pd.DataFrame()
ii=0
IDlist = []
for t1 in Teams:
t1_seed = df_seeds[(df_seeds.TeamID == t1)&(df_seeds.Season == year)].seed_int.values[0]
CompleteDF.loc[ii,'seed'] = t1_seed
[winst1, lossest1] = GetWinsLosses(seasonresults, year, t1)
 CompleteDF.loc[ii,'Season'] = year
CompleteDF.loc[ii,'TeamID'] = t1
CompleteDF.loc[ii,'Wins'] = winst1
CompleteDF.loc[ii,'Losses'] = lossest1
ii+=1 
<filter>
@tf.function
def _mixup(inp, targ):
indice = tf.range(len(inp))
indice = tf.random.shuffle(indice)
 sinp = tf.gather(inp, indice, axis=0)
 starg = tf.gather(targ, indice, axis=0)
 alpha = 0.2
t = tf.compat.v1.distributions.Beta(alpha, alpha ).sample([len(inp)])
 tx = tf.reshape(t, [-1, 1, 1, 1])
 ty = tf.reshape(t, [-1, 1])
 x = inp * tx + sinp *(1-tx)
 y = targ * ty + starg *(1-ty)
 return x, y
BaseDF = seasonresults[seasonresults['Season']==year]
BaseDF.tail()<feature_engineering>
tfrecs = sorted(tf.io.gfile.glob(TRAIN_TFREC + '/*.tfrec'))
parsed_trainval =(tf.data.TFRecordDataset(tfrecs, num_parallel_reads=AUTOTUNE)
.map(_parse_function, num_parallel_calls=AUTOTUNE ).unbatch() 
.filter(_filtTP ).enumerate() )
WinerIDlessthanLoserID = BaseDF['WTeamID']<BaseDF['LTeamID']
BaseDF.loc[WinerIDlessthanLoserID, 'Team1'] = BaseDF['WTeamID']
BaseDF.loc[~WinerIDlessthanLoserID, 'Team1'] = BaseDF['LTeamID']
BaseDF.loc[WinerIDlessthanLoserID, 'Team2'] = BaseDF['LTeamID']
BaseDF.loc[~WinerIDlessthanLoserID, 'Team2'] = BaseDF['WTeamID']
BaseDF.loc[WinerIDlessthanLoserID, 'Result'] = 1
BaseDF.loc[~WinerIDlessthanLoserID, 'Result'] = 0
BaseDF = BaseDF.drop(['DayNum','WScore','LScore','WLoc','NumOT','WTeamID','LTeamID'],axis=1)
<merge>
indices = []
spid = []
recid = []
for i, sample in tqdm(parsed_trainval.prefetch(AUTOTUNE)) :
indices.append(i.numpy())
 spid.append(sample['species_id'].numpy())
 recid.append(sample['recording_id'].numpy().decode() )
FinalDF = pd.merge(left=BaseDF, right=CompleteDF, how='outer', left_on=['Season', 'Team1'], right_on=['Season', 'TeamID'])
 FinalDF.drop(['TeamID'],axis=1,inplace=True)
 FinalDF = FinalDF.rename(columns={'seed':'seedT1','Wins':'WinsT1','Losses':'LossesT1'})
 FinalDF = pd.merge(left=FinalDF, right=CompleteDF, how='right', left_on=['Season', 'Team2'], right_on=['Season', 'TeamID'])
 FinalDF.drop(['TeamID'],axis=1,inplace=True)
 FinalDF = FinalDF.rename(columns={'seed':'seedT2','Wins':'WinsT2','Losses':'LossesT2'})
 FinalDF = FinalDF.dropna()<drop_column>
table = pd.DataFrame({'indices': indices, 'species_id': spid, 'recording_id': recid})
 table
FinalDF['Seed'] = FinalDF['seedT1']-FinalDF['seedT2']
FinalDF['Wins'] = FinalDF['WinsT1']-FinalDF['WinsT2']
FinalDF['Losses'] = FinalDF['LossesT1']-FinalDF['LossesT2']
FinalDF.drop(['seedT1','seedT2','WinsT1','WinsT2','LossesT1','LossesT2'],axis=1,inplace=True)
 FinalDF_full = FinalDF.copy()<train_model>
def create_idx_filter(indice):
@tf.function
def _filt(i, x):
return tf.reduce_any(indice == i)
 return _filt
@tf.function
def _remove_idx(i, x):
return x
X = FinalDF.drop(['Season','Result','Team1','Team2'],axis=1)
 y = FinalDF.Result.values
logreg = LogisticRegression(C=0.1)
 logreg.fit(X, y )<feature_engineering>
def create_train_dataset(batchsize, train_idx):
global parsed_trainval
parsed_train =(parsed_trainval 
.filter(create_idx_filter(train_idx)) 
.map(_remove_idx))
dataset =(parsed_train.cache() 
.shuffle(len(train_idx)) 
.repeat() 
.map(_cut_wav, num_parallel_calls=AUTOTUNE)
.map(_wav_to_spec, num_parallel_calls=AUTOTUNE)
.map(_create_annot, num_parallel_calls=AUTOTUNE)
.map(_preprocess, num_parallel_calls=AUTOTUNE)
.batch(batchsize))
if cfg['model_params']['mixup']:
dataset =(dataset.map(_mixup, num_parallel_calls=AUTOTUNE)
.prefetch(AUTOTUNE))
else:
dataset = dataset.prefetch(AUTOTUNE)
 return dataset
def create_val_dataset(batchsize, val_idx):
global parsed_trainval
parsed_val =(parsed_trainval 
.filter(create_idx_filter(val_idx)) 
.map(_remove_idx))
vdataset =(parsed_val 
.map(_cut_wav_val, num_parallel_calls=AUTOTUNE)
.map(_wav_to_spec, num_parallel_calls=AUTOTUNE)
.map(_create_annot, num_parallel_calls=AUTOTUNE)
.map(_preprocess_val, num_parallel_calls=AUTOTUNE)
.batch(8*strategy.num_replicas_in_sync)
.cache())
 return vdataset
Teams = df_seeds.TeamID.sort_values().unique()
n_test_games = int(len(Teams)*(len(Teams)-1)/2)
 X_test = pd.DataFrame()
ii=0
IDlist = []
for t1 in Teams:
for t2 in Teams:
if t1 < t2:
t1_seed = df_seeds[(df_seeds.TeamID == t1)&(df_seeds.Season == year)].seed_int.values[0]
t2_seed = df_seeds[(df_seeds.TeamID == t2)&(df_seeds.Season == year)].seed_int.values[0]
X_test.loc[ii,'Seed'] = t1_seed - t2_seed
[winst1, lossest1] = GetWinsLosses(seasonresults, year, t1)
 [winst2, lossest2] = GetWinsLosses(seasonresults, year, t2)
 X_test.loc[ii,'Wins'] = winst1-winst2
X_test.loc[ii,'Losses'] = lossest1-lossest2
ii+=1
IDlist.append('{}_{}_{}'.format(year,t1,t2))<prepare_output>
@tf.function
def _one_sample_positive_class_precisions(example):
y_true, y_pred = example
retrieved_classes = tf.argsort(y_pred, direction='DESCENDING')
 class_rankings = tf.argsort(retrieved_classes)
 retrieved_class_true = tf.gather(y_true, retrieved_classes)
 retrieved_cumulative_hits = tf.math.cumsum(tf.cast(retrieved_class_true, tf.float32))
idx = tf.where(y_true)[:, 0]
i = tf.boolean_mask(class_rankings, y_true)
 r = tf.gather(retrieved_cumulative_hits, i)
 c = 1 + tf.cast(i, tf.float32)
 precisions = r / c
dense = tf.scatter_nd(idx[:, None], precisions, [y_pred.shape[0]])
 return dense
class LWLRAP(tf.keras.metrics.Metric):
def __init__(self, num_classes, name='lwlrap'):
super().__init__(name=name)
 self._precisions = self.add_weight(
 name='per_class_cumulative_precision',
shape=[num_classes],
initializer='zeros', 
)
 self._counts = self.add_weight(
 name='per_class_cumulative_count',
shape=[num_classes],
initializer='zeros', 
)
 def update_state(self, y_true, y_pred, sample_weight=None):
precisions = tf.map_fn(
 fn=_one_sample_positive_class_precisions,
elems=(y_true, y_pred),
dtype=(tf.float32), 
)
 increments = tf.cast(precisions > 0, tf.float32)
 total_increments = tf.reduce_sum(increments, axis=0)
 total_precisions = tf.reduce_sum(precisions, axis=0)
 self._precisions.assign_add(total_precisions)
 self._counts.assign_add(total_increments)
 def result(self):
per_class_lwlrap = self._precisions / tf.maximum(self._counts, 1.0)
 per_class_weight = self._counts / tf.reduce_sum(self._counts)
 overall_lwlrap = tf.reduce_sum(per_class_lwlrap * per_class_weight)
 return overall_lwlrap
def reset_states(self):
self._precisions.assign(self._precisions * 0)
 self._counts.assign(self._counts * 0 )
preds = logreg.predict_proba(X_test)[:,1]
clipped_preds = np.clip(preds, 0.01, 0.99)
 df_final_season = pd.DataFrame(columns=['ID','Pred'])
 df_final_season.Pred = clipped_preds
df_final_season.ID = IDlist 
<data_type_conversions>
def _parse_function_test(example_proto):
sample = tf.io.parse_single_example(example_proto, feature_description)
 wav, _ = tf.audio.decode_wav(sample['audio_wav'], desired_channels=1)
 @tf.function
def _cut_audio(i):
_sample = {
'audio_wav': tf.reshape(wav[i*SR*TIME:(i+1)*SR*TIME], [SR*TIME]),
'recording_id': sample['recording_id']
}
return _sample
return tf.map_fn(_cut_audio, tf.range(60//TIME), dtype={
'audio_wav': tf.float32,
'recording_id': tf.string
})
 def inference(model):
tdataset =(tf.data.TFRecordDataset(tf.io.gfile.glob(TEST_TFREC + '/*.tfrec'), num_parallel_reads=AUTOTUNE)
.map(_parse_function_test, num_parallel_calls=AUTOTUNE ).unbatch() 
.map(_wav_to_spec, num_parallel_calls=AUTOTUNE)
.map(_preprocess_test, num_parallel_calls=AUTOTUNE)
.batch(128*(60//TIME)).prefetch(AUTOTUNE))
rec_ids = []
probs = []
for inp, rec_id in tqdm(tdataset):
with strategy.scope() :
pred = model.predict_on_batch(tf.reshape(inp, [-1, HEIGHT, WIDTH, 3]))
prob = tf.sigmoid(pred)
 prob = tf.reduce_max(tf.reshape(prob, [-1, 60//TIME, CLASS_N]), axis=1)
 rec_id_stack = tf.reshape(rec_id, [-1, 60//TIME])
 for rec in rec_id.numpy() :
assert len(np.unique(rec)) == 1
rec_ids.append(rec_id_stack.numpy() [:,0])
 probs.append(prob.numpy())
 crec_ids = np.concatenate(rec_ids)
 cprobs = np.concatenate(probs)
 sub = pd.DataFrame({
'recording_id': list(map(lambda x: x.decode() , crec_ids.tolist())) ,
**{f's{i}': cprobs[:,i] for i in range(CLASS_N)}
})
 sub = sub.sort_values('recording_id')
 return sub
BaseDF = tourresults
WinerIDlessthanLoserID = BaseDF['WTeamID']<BaseDF['LTeamID']
BaseDF.loc[WinerIDlessthanLoserID, 'Team1'] = BaseDF['WTeamID']
BaseDF.loc[~WinerIDlessthanLoserID, 'Team1'] = BaseDF['LTeamID']
BaseDF.loc[WinerIDlessthanLoserID, 'Team2'] = BaseDF['LTeamID']
BaseDF.loc[~WinerIDlessthanLoserID, 'Team2'] = BaseDF['WTeamID']
BaseDF.loc[WinerIDlessthanLoserID, 'Result'] = 1
BaseDF.loc[~WinerIDlessthanLoserID, 'Result'] = 0
BaseDF['Team1'] = BaseDF['Team1'].astype(int)
 BaseDF['Team2'] = BaseDF['Team2'].astype(int)
 BaseDF.loc[:,'ID']=BaseDF['Season'].astype(str)+'_'+BaseDF['Team1'].astype(str)+'_'+BaseDF['Team2'].astype(str)
 isSeason = BaseDF['Season'] == year
BaseDF = BaseDF[isSeason]
BaseDF = BaseDF.drop(['DayNum','WScore','LScore','WLoc','NumOT','WTeamID','LTeamID','Team1','Team2','Season'],axis=1 )<feature_engineering>
def train_and_inference(splits, split_id):
batchsize = cfg['model_params']['batchsize_per_tpu'] * strategy.num_replicas_in_sync
print("batchsize", batchsize)
 loss_fn = cfg['model_params']['loss']['fn'](from_logits=True, **cfg['model_params']['loss']['params'])
 idx_train_tf = tf.constant(splits[split_id][0])
 idx_val_tf = tf.constant(splits[split_id][1])
 dataset = create_train_dataset(batchsize, idx_train_tf)
 vdataset = create_val_dataset(batchsize, idx_val_tf)
 optimizer = cfg['model_params']['optim']['fn'](**cfg['model_params']['optim']['params'])
 model = create_model()
with strategy.scope() :
model.compile(optimizer=optimizer, loss=loss_fn, metrics=[LWLRAP(CLASS_N)])
 history = model.fit(dataset,
steps_per_epoch=cfg['model_params']['iteration_per_epoch'],
epochs=cfg['model_params']['epoch'],
validation_data=vdataset,
callbacks=[
tf.keras.callbacks.ReduceLROnPlateau(
 'val_lwlrap', patience=10 
),
tf.keras.callbacks.ModelCheckpoint(
 filepath='model_best_%d.h5' % split_id,
save_weights_only=True,
monitor='val_lwlrap',
mode='max',
save_best_only=True),
])
 plot_history(history, 'history_%d.png' % split_id)
 model.load_weights('model_best_%d.h5' % split_id)
 return inference(model), history
BaseDF['ID']=IDlist
BaseDF['Result']=0<merge>
sub = sum(
 map(
 lambda i: train_and_inference(splits, i)[0].set_index('recording_id'),
range(len(splits)) 
)
 ).reset_index()
<feature_engineering><EOS> 
sub.to_csv("submission.csv", index=False )