Kaggle competition: Digit Recognizer
def seed_everything(seed=0):
random.seed(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 os.environ['TF_DETERMINISTIC_OPS'] = '1'
seed = 42
seed_everything(seed)
 warnings.filterwarnings('ignore' )<define_variables>
train = pd.read_csv(".. /input/train.csv")
 print(train.shape)
 train.head()
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1)) for filename in filenames]
return np.sum(n)
 TRAIN_DATA_DIR = 'rfcx-audio-detection'
TRAIN_GCS_PATH = KaggleDatasets().get_gcs_path(TRAIN_DATA_DIR)
 FILENAMES = tf.io.gfile.glob(TRAIN_GCS_PATH + '/tp*.tfrec')
 TEST_DATA_DIR = 'rfcx-species-audio-detection'
TEST_GCS_PATH = KaggleDatasets().get_gcs_path(TEST_DATA_DIR)
 TEST_FILES = tf.io.gfile.glob(TEST_GCS_PATH + '/tfrecords/test/*.tfrec')
 no_of_training_samples = count_data_items(FILENAMES)
 print('num_training_samples are', no_of_training_samples )<init_hyperparams>
test= pd.read_csv(".. /input/test.csv")
 print(test.shape)
 test.head()
CUT = 10
TIME = 10
EPOCHS = 25
GLOBAL_BATCH_SIZE = 4 * REPLICAS
LEARNING_RATE = 0.0015
WARMUP_LEARNING_RATE = 1e-5
WARMUP_EPOCHS = int(EPOCHS*0.1)
 PATIENCE = 8
STEPS_PER_EPOCH = 64
N_FOLDS = 5
NUM_TRAINING_SAMPLES = no_of_training_samples
class params:
sample_rate = 48000
stft_window_seconds: float = 0.025
stft_hop_seconds: float = 0.005
frame_length: int = 1200
mel_bands: int = 512
mel_min_hz: float = 50.0
mel_max_hz: float = 24000.0
log_offset: float = 0.001
patch_window_seconds: float = 0.96
patch_hop_seconds: float = 0.48
patch_frames = int(round(patch_window_seconds / stft_hop_seconds))
patch_bands = mel_bands
height = mel_bands
width = 2000
num_classes: int = 24
dropout = 0.35
classifier_activation: str = 'sigmoid' 
<feature_engineering>
Y_train = train["label"]
X_train = train.drop(labels = ["label"],axis = 1)
 X_train = X_train / 255.0
X_test = test / 255.0
X_train = X_train.values.reshape(-1,28,28,1)
 X_test = X_test.values.reshape(-1,28,28,1)
 Y_train = to_categorical(Y_train, num_classes = 10 )
feature_description = {
'wav': tf.io.FixedLenFeature([], tf.string),
'recording_id': tf.io.FixedLenFeature([], tf.string),
'target' : tf.io.FixedLenFeature([], tf.float32),
'song_id': tf.io.FixedLenFeature([], tf.float32),
'tmin' : tf.io.FixedLenFeature([], tf.float32),
'fmin' : tf.io.FixedLenFeature([], tf.float32),
'tmax' : tf.io.FixedLenFeature([], tf.float32),
'fmax' : tf.io.FixedLenFeature([], tf.float32),
}
feature_dtype = {
'wav': tf.float32,
'recording_id': tf.string,
'target': tf.float32,
'song_id': tf.float32,
't_min': tf.float32,
'f_min': tf.float32,
't_max': tf.float32,
'f_max':tf.float32,
}<normalization>
datagen = ImageDataGenerator(
 rotation_range=10,
zoom_range = 0.10,
width_shift_range=0.1,
height_shift_range=0.1 )
def waveform_to_log_mel_spectrogram(waveform,target_or_rec_id):

 window_length_samples = int(
 round(params.sample_rate * params.stft_window_seconds))
hop_length_samples = int(
 round(params.sample_rate * params.stft_hop_seconds))
fft_length = 2 ** int(np.ceil(np.log(window_length_samples)/ np.log(2.0)))
 num_spectrogram_bins = fft_length // 2 + 1
magnitude_spectrogram = tf.abs(tf.signal.stft(
 signals=waveform,
frame_length=params.frame_length,
frame_step=hop_length_samples,
fft_length= fft_length))
linear_to_mel_weight_matrix = tf.signal.linear_to_mel_weight_matrix(
 num_mel_bins=params.mel_bands,
num_spectrogram_bins=num_spectrogram_bins,
sample_rate=params.sample_rate,
lower_edge_hertz=params.mel_min_hz,
upper_edge_hertz=params.mel_max_hz)
 mel_spectrogram = tf.matmul(
 magnitude_spectrogram, linear_to_mel_weight_matrix)
 log_mel = tf.math.log(mel_spectrogram + params.log_offset)
 log_mel = tf.transpose(log_mel)
 log_mel_spectrogram = tf.reshape(log_mel , [tf.shape(log_mel)[0] ,tf.shape(log_mel)[1],1])
 spectrogram_hop_length_samples = int(
 round(params.sample_rate * params.stft_hop_seconds))
spectrogram_sample_rate = params.sample_rate / spectrogram_hop_length_samples
patch_window_length_samples = int(
 round(spectrogram_sample_rate * params.patch_window_seconds))
patch_hop_length_samples = int(
 round(spectrogram_sample_rate * params.patch_hop_seconds))
features = tf.signal.frame(
 signal=log_mel_spectrogram,
frame_length=patch_window_length_samples,
frame_step=patch_hop_length_samples,
axis=0)
 return log_mel_spectrogram, target_or_rec_id<categorify>
nets = 10
model = [0] *nets
for j in range(nets):
model[j] = Sequential()
model[j].add(Conv2D(32, kernel_size = 3, activation='relu', input_shape =(28, 28, 1)))
 model[j].add(BatchNormalization())
 model[j].add(Conv2D(32, kernel_size = 3, activation='relu'))
model[j].add(BatchNormalization())
 model[j].add(Conv2D(32, kernel_size = 5, strides=2, padding='same', activation='relu'))
model[j].add(BatchNormalization())
 model[j].add(Dropout(0.4))
model[j].add(Conv2D(64, kernel_size = 3, activation='relu'))
model[j].add(BatchNormalization())
 model[j].add(Conv2D(64, kernel_size = 3, activation='relu'))
model[j].add(BatchNormalization())
 model[j].add(Conv2D(64, kernel_size = 5, strides=2, padding='same', activation='relu'))
model[j].add(BatchNormalization())
 model[j].add(Dropout(0.4))
model[j].add(Conv2D(128, kernel_size = 4, activation='relu'))
model[j].add(BatchNormalization())
 model[j].add(Flatten())
 model[j].add(Dropout(0.4))
model[j].add(Dense(10, activation='softmax'))
model[j].compile(optimizer="adam", loss="categorical_crossentropy", metrics=["accuracy"] )
def frequency_masking(mel_spectrogram):
frequency_masking_para = 80,
frequency_mask_num = 2
fbank_size = tf.shape(mel_spectrogram)
 n, v = fbank_size[0], fbank_size[1]
for i in range(frequency_mask_num):
f = tf.random.uniform([], minval=0, maxval= tf.squeeze(frequency_masking_para), dtype=tf.int32)
 v = tf.cast(v, dtype=tf.int32)
 f0 = tf.random.uniform([], minval=0, maxval= tf.squeeze(v-f), dtype=tf.int32)
 mask = tf.concat(( tf.ones(shape=(n, v - f0 - f,1)) ,
tf.zeros(shape=(n, f,1)) ,
tf.ones(shape=(n, f0,1)) , 
),1)
 mel_spectrogram = mel_spectrogram * mask
return tf.cast(mel_spectrogram, dtype=tf.float32)
 def time_masking(mel_spectrogram):
time_masking_para = 40,
time_mask_num = 1
fbank_size = tf.shape(mel_spectrogram)
 n, v = fbank_size[0], fbank_size[1]
for i in range(time_mask_num):
t = tf.random.uniform([], minval=0, maxval=tf.squeeze(time_masking_para), dtype=tf.int32)
 t0 = tf.random.uniform([], minval=0, maxval= n-t, dtype=tf.int32)
 mask = tf.concat(( tf.ones(shape=(n-t0-t, v,1)) ,
tf.zeros(shape=(t, v,1)) ,
tf.ones(shape=(t0, v,1)) , 
), 0)
 mel_spectrogram = mel_spectrogram * mask
return tf.cast(mel_spectrogram, dtype=tf.float32)
 def random_brightness(image):
return tf.image.random_brightness(image, 0.2)
 def random_gamma(image):
return tf.image.random_contrast(image, lower = 0.1, upper = 0.3)
 def random_flip_right(image):
return tf.image.random_flip_left_right(image)
 def random_flip_up_down(image):
return tf.image.random_flip_left_right(image)
 available_ops = [
frequency_masking ,
time_masking,
random_brightness,
random_flip_up_down,
random_flip_right
]
def apply_augmentation(image, target):
num_layers = int(np.random.uniform(low = 0, high = 3))
for layer_num in range(num_layers):
op_to_select = tf.random.uniform([], maxval=len(available_ops), dtype=tf.int32, seed = seed)
 for(i, op_name)in enumerate(available_ops):
image = tf.cond(
 tf.equal(i, op_to_select),
lambda selected_func=op_name,: selected_func(
 image),
lambda: image)
 return image, target<categorify>
annealer = LearningRateScheduler(lambda x: 1e-3 * 0.95 ** x)
 history = [0] * nets
epochs = 45
for j in range(nets):
X_train2, X_val2, Y_train2, Y_val2 = train_test_split(X_train, Y_train, test_size = 0.1)
 history[j] = model[j].fit_generator(datagen.flow(X_train2,Y_train2, batch_size=64),
epochs = epochs, steps_per_epoch = X_train2.shape[0]//64,
validation_data =(X_val2,Y_val2), callbacks=[annealer], verbose=0)
 print("CNN {0:d}: Epochs={1:d}, Train accuracy={2:.5f}, Validation accuracy={3:.5f}".format(
 j+1,epochs,max(history[j].history['acc']),max(history[j].history['val_acc'])) )
<create_dataframe><EOS> 
results = np.zeros(( X_test.shape[0],10))
for j in range(nets):
results = results + model[j].predict(X_test)
 results = np.argmax(results,axis = 1)
 results = pd.Series(results,name="Label")
 submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),results],axis = 1)
 submission.to_csv("MNIST-CNN-ENSEMBLE.csv",index=False )