Kaggle competition: Digit Recognizer
def load_test_file(f):
wav, sr = librosa.load('/kaggle/input/rfcx-species-audio-detection/test/' + f, sr=None)
 segments = len(wav)/ length
segments = int(np.ceil(segments))
mel_array = []
for i in range(0, segments):
if(i + 1)* length > len(wav):
slice = wav[len(wav)- length:len(wav)]
else:
slice = wav[i * length:(i + 1)* length]
spec=librosa.feature.melspectrogram(slice, sr=sr,n_fft=fft,hop_length=hop,fmin=fmin,fmax=fmax)
 spec_db=librosa.power_to_db(spec,top_db=80)
 img = spec_to_image(spec_db)
 mel_spec = np.stack(( img, img, img))
mel_array.append(mel_spec)
 return mel_array<load_pretrained>
import pandas as pd
import numpy as np
del audio_data
members = []
for i in range(nfold):
model = get_model()
model.load_state_dict(torch.load('/kaggle/working/model'+str(i)+'.pt'))
model.eval()
members.append(model )<save_to_csv>
import pandas as pd
import numpy as np
if save_to_disk == 0:
for f in os.listdir('/kaggle/working/'):
os.remove('/kaggle/working/' + f)
 print('Starting prediction loop')
 with open('submission.csv', 'w', newline='')as csvfile:
submission_writer = csv.writer(csvfile, delimiter=',')
 submission_writer.writerow(['recording_id','s0','s1','s2','s3','s4','s5','s6','s7','s8','s9','s10','s11',
's12','s13','s14','s15','s16','s17','s18','s19','s20','s21','s22','s23'])
 test_files = os.listdir('/kaggle/input/rfcx-species-audio-detection/test/')
 print(len(test_files))
for i in range(0, len(test_files)) :
data = load_test_file(test_files[i])
 data = torch.tensor(data)
 data = data.float()
if torch.cuda.is_available() :
data = data.cuda()
output_list = []
for m in members:
output = m(data)
 maxed_output = torch.max(output, dim=0)[0]
maxed_output = maxed_output.cpu().detach()
output_list.append(maxed_output)
 avg_maxed_output = torch.mean(torch.stack(output_list), dim=0)
 file_id = str.split(test_files[i], '.')[0]
write_array = [file_id]
for out in avg_maxed_output:
write_array.append(out.item())
 submission_writer.writerow(write_array)
 if i % 100 == 0 and i > 0:
print('Predicted for ' + str(i)+ ' of ' + str(len(test_files)+ 1)+ ' files')
 print('Submission generated' )<install_modules>
import tensorflow as tf
from tensorflow import keras
! pip install -q efficientnet<import_modules>
train = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')
 train.head()
import math, os, re, warnings, random , time
from collections import namedtuple
import tensorflow as tf
import numpy as np
import pandas as pd
import librosa
from kaggle_datasets import KaggleDatasets
import matplotlib.pyplot as plt
from IPython.display import Audio
from tensorflow.keras import Model, layers , optimizers
from sklearn.model_selection import KFold
import tensorflow.keras.backend as K
from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, LearningRateScheduler
from tensorflow.keras.layers import GlobalAveragePooling2D, Input, Dense, Dropout, GaussianNoise, concatenate
from tensorflow.keras.applications import ResNet50
import efficientnet.keras as efn
import seaborn as sns<set_options>
test = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')
 test.head()
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print(f'Running on TPU {tpu.master() }')
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
AUTO = tf.data.experimental.AUTOTUNE
REPLICAS = strategy.num_replicas_in_sync
print(f'REPLICAS: {REPLICAS}' )<define_variables>
X_train /= 255
test /= 255
def seed_everything(seed=0):
random.seed(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 os.environ['TF_DETERMINISTIC_OPS'] = '1'
seed = 42
seed_everything(seed)
 warnings.filterwarnings('ignore' )<define_variables>
X_train = X_train.values.reshape(-1,28,28,1)
 test = test.values.reshape(-1,28,28,1 )
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1)) for filename in filenames]
return np.sum(n)
 TRAIN_DATA_DIR = 'rfcx-audio-detection'
TRAIN_GCS_PATH = KaggleDatasets().get_gcs_path(TRAIN_DATA_DIR)
 FILENAMES = tf.io.gfile.glob(TRAIN_GCS_PATH + '/tp*.tfrec')
 TEST_DATA_DIR = 'rfcx-species-audio-detection'
TEST_GCS_PATH = KaggleDatasets().get_gcs_path(TEST_DATA_DIR)
 TEST_FILES = tf.io.gfile.glob(TEST_GCS_PATH + '/tfrecords/test/*.tfrec')
 no_of_training_samples = count_data_items(FILENAMES)
 print('num_training_samples are', no_of_training_samples )<define_variables>
Y_train = to_categorical(y_train,num_classes = 10 )
CUT = 10
TIME = 10
EPOCHS = 25
GLOBAL_BATCH_SIZE = 4 * REPLICAS
LEARNING_RATE = 0.0015
WARMUP_LEARNING_RATE = 1e-5
WARMUP_EPOCHS = int(EPOCHS*0.1)
 PATIENCE = 10
STEPS_PER_EPOCH = 64
N_FOLDS = 5
NUM_TRAINING_SAMPLES = no_of_training_samples
class params:
sample_rate = 48000
stft_window_seconds: float = 0.025
stft_hop_seconds: float = 0.005
frame_length: int = 1200
mel_bands: int = 256
mel_min_hz: float = 50.0
mel_max_hz: float = 16000.0
log_offset: float = 0.001
patch_bands = mel_bands
conv_padding: str = 'same'
batchnorm_center: bool = True
batchnorm_scale: bool = False
batchnorm_epsilon: float = 1e-4
num_classes: int = 24
dropout = 0.40
classifier_activation: str = 'sigmoid'
height = mel_bands
width = 1024<feature_engineering>
random_seed=2
X_train, X_val, Y_train, Y_val = train_test_split(X_train,Y_train,test_size = 0.1, random_state=random_seed )
feature_description = {
'wav': tf.io.FixedLenFeature([], tf.string),
'recording_id': tf.io.FixedLenFeature([], tf.string),
'target' : tf.io.FixedLenFeature([], tf.float32),
'song_id': tf.io.FixedLenFeature([], tf.float32),
'tmin' : tf.io.FixedLenFeature([], tf.float32),
'fmin' : tf.io.FixedLenFeature([], tf.float32),
'tmax' : tf.io.FixedLenFeature([], tf.float32),
'fmax' : tf.io.FixedLenFeature([], tf.float32),
}
feature_dtype = {
'wav': tf.float32,
'recording_id': tf.string,
'target': tf.float32,
'song_id': tf.float32,
't_min': tf.float32,
'f_min': tf.float32,
't_max': tf.float32,
'f_max':tf.float32,
}<normalization>
from keras.models import Sequential
from keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPool2D
from keras.optimizers import RMSprop
from keras.preprocessing.image import ImageDataGenerator
from keras.callbacks import ReduceLROnPlateau
def waveform_to_log_mel_spectrogram(waveform,target_or_rec_id):

 window_length_samples = int(
 round(params.sample_rate * params.stft_window_seconds))
hop_length_samples = int(
 round(params.sample_rate * params.stft_hop_seconds))
fft_length = 2 ** int(np.ceil(np.log(window_length_samples)/ np.log(2.0)))
 num_spectrogram_bins = fft_length // 2 + 1
magnitude_spectrogram = tf.abs(tf.signal.stft(
 signals=waveform,
frame_length=params.frame_length,
frame_step=hop_length_samples,
fft_length= fft_length))
linear_to_mel_weight_matrix = tf.signal.linear_to_mel_weight_matrix(
 num_mel_bins=params.mel_bands,
num_spectrogram_bins=num_spectrogram_bins,
sample_rate=params.sample_rate,
lower_edge_hertz=params.mel_min_hz,
upper_edge_hertz=params.mel_max_hz)
 mel_spectrogram = tf.matmul(
 magnitude_spectrogram, linear_to_mel_weight_matrix)
 log_mel = tf.math.log(mel_spectrogram + params.log_offset)
 log_mel = tf.transpose(log_mel)
 log_mel_spectrogram = tf.reshape(log_mel , [tf.shape(log_mel)[0] ,tf.shape(log_mel)[1],1])
 return log_mel_spectrogram, target_or_rec_id<categorify>
cnn = Sequential()
cnn.add(Conv2D(filters=32,kernel_size=(5,5),padding='Same',activation='relu', input_shape=(28,28,1)))
 cnn.add(Conv2D(filters=32,kernel_size=(5,5),padding='Same',activation='relu'))
cnn.add(MaxPool2D(pool_size=(2,2)))
 cnn.add(Dropout(0.25))
cnn.add(Conv2D(filters=64,kernel_size=(3,3),padding='Same',activation='relu'))
cnn.add(Conv2D(filters=64,kernel_size=(3,3),padding='Same',activation='relu'))
cnn.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))
 cnn.add(Dropout(0.25))
cnn.add(Flatten())
 cnn.add(Dense(units=256, activation='relu'))
cnn.add(Dropout(0.25))
cnn.add(Dense(units=10, activation='softmax'))
def frequency_masking(mel_spectrogram):
frequency_masking_para = 80,
frequency_mask_num = 2
fbank_size = tf.shape(mel_spectrogram)
 n, v = fbank_size[0], fbank_size[1]
for i in range(frequency_mask_num):
f = tf.random.uniform([], minval=0, maxval= tf.squeeze(frequency_masking_para), dtype=tf.int32)
 v = tf.cast(v, dtype=tf.int32)
 f0 = tf.random.uniform([], minval=0, maxval= tf.squeeze(v-f), dtype=tf.int32)
 mask = tf.concat(( tf.ones(shape=(n, v - f0 - f,1)) ,
tf.zeros(shape=(n, f,1)) ,
tf.ones(shape=(n, f0,1)) , 
),1)
 mel_spectrogram = mel_spectrogram * mask
return tf.cast(mel_spectrogram, dtype=tf.float32)
 def time_masking(mel_spectrogram):
time_masking_para = 40,
time_mask_num = 1
fbank_size = tf.shape(mel_spectrogram)
 n, v = fbank_size[0], fbank_size[1]
for i in range(time_mask_num):
t = tf.random.uniform([], minval=0, maxval=tf.squeeze(time_masking_para), dtype=tf.int32)
 t0 = tf.random.uniform([], minval=0, maxval= n-t, dtype=tf.int32)
 mask = tf.concat(( tf.ones(shape=(n-t0-t, v,1)) ,
tf.zeros(shape=(t, v,1)) ,
tf.ones(shape=(t0, v,1)) , 
), 0)
 mel_spectrogram = mel_spectrogram * mask
return tf.cast(mel_spectrogram, dtype=tf.float32)
 def random_brightness(image):
return tf.image.random_brightness(image, 0.2)
 def random_gamma(image):
return tf.image.random_contrast(image, lower = 0.1, upper = 0.3)
 def random_flip_right(image):
return tf.image.random_flip_left_right(image)
 def random_flip_up_down(image):
return tf.image.random_flip_left_right(image)
 available_ops = [
frequency_masking ,
time_masking,
random_brightness,
random_flip_up_down,
random_flip_right
]
def apply_augmentation(image, target):
num_layers = int(np.random.uniform(low = 0, high = 3))
for layer_num in range(num_layers):
op_to_select = tf.random.uniform([], maxval=len(available_ops), dtype=tf.int32)
 for(i, op_name)in enumerate(available_ops):
image = tf.cond(
 tf.equal(i, op_to_select),
lambda selected_func=op_name,: selected_func(
 image),
lambda: image)
 return image, target<categorify>
optimizer = RMSprop(lr=0.001,rho=0.9,epsilon=1e-08,decay=0.0 )
def preprocess(image, target_or_rec_id):
image = tf.image.grayscale_to_rgb(image)
 image = tf.image.resize(image, [params.height,params.width])
 image = tf.image.per_image_standardization(image)
 return image , target_or_rec_id
def read_labeled_tfrecord(example_proto):
sample = tf.io.parse_single_example(example_proto, feature_description)
 wav, _ = tf.audio.decode_wav(sample['wav'], desired_channels=1)
 target = tf.cast(sample['target'],tf.float32)
 target = tf.squeeze(tf.one_hot([target,], depth = params.num_classes), axis = 0)
 tmin = tf.cast(sample['tmin'], tf.float32)
 fmin = tf.cast(sample['fmin'], tf.float32)
 tmax = tf.cast(sample['tmax'], tf.float32)
 fmax = tf.cast(sample['fmax'], tf.float32)
 tmax_s = tmax * tf.cast(params.sample_rate, tf.float32)
 tmin_s = tmin * tf.cast(params.sample_rate, tf.float32)
 cut_s = tf.cast(CUT * params.sample_rate, tf.float32)
 all_s = tf.cast(60 * params.sample_rate, tf.float32)
 tsize_s = tmax_s - tmin_s
cut_min = tf.cast(
 tf.maximum(0.0,
tf.minimum(tmin_s -(cut_s - tsize_s)/ 2,
tf.minimum(tmax_s +(cut_s - tsize_s)/ 2, all_s)- cut_s)
), tf.int32 
)
 cut_max = cut_min + CUT * params.sample_rate
wav = tf.squeeze(wav[cut_min : cut_max])
 return wav, target
def read_unlabeled_tfrecord(example):
feature_description = {
'recording_id': tf.io.FixedLenFeature([], tf.string),
'audio_wav': tf.io.FixedLenFeature([], tf.string),
}
sample = tf.io.parse_single_example(example, feature_description)
 wav, _ = tf.audio.decode_wav(sample['audio_wav'], desired_channels=1)
 recording_id = tf.reshape(tf.cast(sample['recording_id'] , tf.string), [1])
 def _cut_audio(i):
_sample = {
'audio_wav': tf.reshape(wav[i*params.sample_rate*TIME:(i+1)*params.sample_rate*TIME], [params.sample_rate*TIME]),
'recording_id': sample['recording_id']
}
return _sample
return tf.map_fn(_cut_audio, tf.range(60//TIME), dtype={
'audio_wav': tf.float32,
'recording_id': tf.string
} )<create_dataframe>
cnn.compile(optimizer = optimizer, loss ='categorical_crossentropy', metrics=['accuracy'] )
def load_dataset(filenames, labeled = True, ordered = False , training = True):
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads = AUTO)
 dataset = dataset.map(read_labeled_tfrecord , num_parallel_calls = AUTO)
 dataset = dataset.map(waveform_to_log_mel_spectrogram , num_parallel_calls = AUTO)
 dataset = dataset.map(preprocess, num_parallel_calls = AUTO)
 return dataset<prepare_x_and_y>
learning_rate_reduction = ReduceLROnPlateau(moniter = 'val_acc',
patience = 3,
verbose = 1,
factor=0.5,
min_lr = 0.00001 )
def get_dataset(filenames, training = True):
if training:
dataset = load_dataset(filenames , training = True)
 dataset = dataset.shuffle(256 ).repeat()
dataset = dataset.batch(GLOBAL_BATCH_SIZE, drop_remainder = True)
 else:
dataset = load_dataset(filenames , training = False)
 dataset = dataset.repeat().batch(GLOBAL_BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset<groupby>
epochs = 10
batch_size = 86
@tf.function
def _one_sample_positive_class_precisions(example):
y_true, y_pred = example
y_true = tf.reshape(y_true, tf.shape(y_pred))
retrieved_classes = tf.argsort(y_pred, direction='DESCENDING')
 class_rankings = tf.argsort(retrieved_classes)
 retrieved_class_true = tf.gather(y_true, retrieved_classes)
 retrieved_cumulative_hits = tf.math.cumsum(tf.cast(retrieved_class_true, tf.float32))
idx = tf.where(y_true)[:, 0]
i = tf.boolean_mask(class_rankings, y_true)
 r = tf.gather(retrieved_cumulative_hits, i)
 c = 1 + tf.cast(i, tf.float32)
 precisions = r / c
dense = tf.scatter_nd(idx[:, None], precisions, [y_pred.shape[0]])
 return dense
class LWLRAP(tf.keras.metrics.Metric):
def __init__(self, num_classes, name='lwlrap'):
super().__init__(name=name)
 self._precisions = self.add_weight(
 name='per_class_cumulative_precision',
shape=[num_classes],
initializer='zeros', 
)
 self._counts = self.add_weight(
 name='per_class_cumulative_count',
shape=[num_classes],
initializer='zeros', 
)
 def update_state(self, y_true, y_pred, sample_weight=None):
precisions = tf.map_fn(
 fn=_one_sample_positive_class_precisions,
elems=(y_true, y_pred),
dtype=(tf.float32), 
)
 increments = tf.cast(precisions > 0, tf.float32)
 total_increments = tf.reduce_sum(increments, axis=0)
 total_precisions = tf.reduce_sum(precisions, axis=0)
 self._precisions.assign_add(total_precisions)
 self._counts.assign_add(total_increments)
 def result(self):
per_class_lwlrap = self._precisions / tf.maximum(self._counts, 1.0)
 per_class_weight = self._counts / tf.reduce_sum(self._counts)
 overall_lwlrap = tf.reduce_sum(per_class_lwlrap * per_class_weight)
 return overall_lwlrap
def reset_states(self):
self._precisions.assign(self._precisions * 0)
 self._counts.assign(self._counts * 0 )<normalization>
learning_rate_base = LEARNING_RATE
total_steps = STEPS_PER_EPOCH * EPOCHS
warmup_learning_rate = WARMUP_LEARNING_RATE
warmup_steps= WARMUP_EPOCHS * STEPS_PER_EPOCH
@tf.function
def cosine_decay_with_warmup(global_step,
hold_base_rate_steps=0):
if total_steps < warmup_steps:
raise ValueError('total_steps must be larger or equal to '
'warmup_steps.')
 learning_rate = 0.5 * learning_rate_base *(1 + tf.cos(
 np.pi * 
(tf.cast(global_step, tf.float32)- warmup_steps - hold_base_rate_steps 
)/ float(total_steps - warmup_steps - hold_base_rate_steps)))
 if hold_base_rate_steps > 0:
learning_rate = tf.where(
 global_step > warmup_steps + hold_base_rate_steps,
learning_rate, learning_rate_base)
 if warmup_steps > 0:
if learning_rate_base < warmup_learning_rate:
raise ValueError('learning_rate_base must be larger or equal to '
'warmup_learning_rate.')
 slope =(learning_rate_base - warmup_learning_rate)/ warmup_steps
warmup_rate = slope * tf.cast(global_step,
tf.float32)+ warmup_learning_rate
learning_rate = tf.where(global_step < warmup_steps, warmup_rate,
learning_rate)
 return tf.where(global_step > total_steps, 0.0, learning_rate,
name='learning_rate')
 rng = [i for i in range(int(EPOCHS * STEPS_PER_EPOCH)) ]
WARMUP_STEPS = int(WARMUP_EPOCHS * STEPS_PER_EPOCH)
 y = [cosine_decay_with_warmup(x)for x in rng]
sns.set(style='whitegrid')
 fig, ax = plt.subplots(figsize=(20, 6))
plt.plot(rng, y )<choose_model_class>
datagen = ImageDataGenerator(featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization = False,
samplewise_std_normalization = False,
zca_whitening=False,
rotation_range=10,
zoom_range=0.1,
width_shift_range = 0.1,
height_shift_range = 0.1,
horizontal_flip = False,
vertical_flip = False)
 datagen.fit(X_train )
class RFCX_MODEL(tf.keras.Model):
def __init__(self):
super(RFCX_MODEL , self ).__init__()
self.gaussian_noise = GaussianNoise(0.05)
 self.resnet_model = ResNet50(include_top=False, weights='imagenet')
 self.model_output = GlobalAveragePooling2D()
self.dropout = Dropout(params.dropout)
 self.predictions = Dense(params.num_classes, activation = params.classifier_activation)
 def call(self, inputs):
noisy_input = self.gaussian_noise(inputs)
 resnet_output = self.resnet_model(noisy_input)
 x = self.model_output(resnet_output)
 x = self.dropout(x)
 x = self.predictions(x)
 return x<split>
history = cnn.fit_generator(datagen.flow(X_train,Y_train,batch_size=batch_size),
epochs =epochs, validation_data =(X_val, Y_val),
verbose=2, steps_per_epoch=X_train.shape[0]//batch_size,
callbacks = [learning_rate_reduction] )
def train_one_fold(train_dataset, valid_dataset):
print('Start fine-tuning!', flush=True)
 train_dist_dataset = strategy.experimental_distribute_dataset(train_dataset)
 valid_dist_dataset = strategy.experimental_distribute_dataset(valid_dataset)
 start_time = epoch_start_time = time.time()
print("Steps per epoch:", STEPS_PER_EPOCH)
 History = namedtuple('History', 'history')
 history = History(history={'train_loss': [], 'val_loss': [], 'train_lwlrap' : [], 'val_lwlrap' : []})
 train_iterator = iter(train_dist_dataset)
 val_iterator = iter(valid_dist_dataset)
 steps = 0
best_val_lwlrap = 0
for epoch in range(EPOCHS):
print('
EPOCH {:d}/{:d}'.format(epoch+1, EPOCHS))
train_multiple_steps(train_iterator , tf.convert_to_tensor(STEPS_PER_EPOCH))
steps += STEPS_PER_EPOCH
if(steps // STEPS_PER_EPOCH)> epoch:
val_steps = 0
val_multiple_steps(val_iterator, tf.convert_to_tensor(VAL_STEPS))
val_steps += VAL_STEPS
history.history['train_loss'].append(train_loss.result().numpy() /(STEPS_PER_EPOCH))
history.history['val_loss'].append(( val_loss.result().numpy() / val_steps))
history.history['train_lwlrap'].append(train_lwlrap.result().numpy())
 history.history['val_lwlrap'].append(val_lwlrap.result().numpy())
 epoch_time = time.time() - epoch_start_time
print('time: {:0.1f}s'.format(epoch_time),
'train_loss: {:0.4f}'.format(history.history['train_loss'][-1]),
'val_loss: {:0.4f}'.format(history.history['val_loss'][-1]),
'train_lwlrap: {:0.4f}'.format(history.history['train_lwlrap'][-1]),
'val_lwlrap: {:0.4f}'.format(history.history['val_lwlrap'][-1]),
'lr : {:0.6f}'.format(cosine_decay_with_warmup(steps)) 
)
 if history.history['val_lwlrap'][-1] >= best_val_lwlrap:
best_val_lwlrap = history.history['val_lwlrap'][-1]
model.save_weights(model_path)
 print(f'Saved model weights at "{model_path}"')
 patience_cnt = 1
else:
patience_cnt += 1
if patience_cnt > PATIENCE:
print(f'Epoch {epoch:05d}: early stopping')
 break
epoch_start_time = time.time()
train_loss.reset_states()
val_loss.reset_states()
train_lwlrap.reset_states()
val_lwlrap.reset_states()
history_list.append(history )<split>
results = cnn.predict(test)
 results = np.argmax(results,axis = 1)
 results = pd.Series(results,name="Label" )
<create_dataframe><EOS> 
submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),results],axis = 1)
 submission.to_csv("cnn_mnist_datagen.csv",index=False)